Skip to content

Commit 2c5fd87

Browse files
sparkprimesparkprime
committed
Fix #8 #18 
1 parent b13e6e1 commit 2c5fd87

13 files changed

+8488
-7929
lines changed

‎_jsonnet.c

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,13 @@ limitations under the License.
2323

2424
static PyObject* evaluate_file(PyObject* self, PyObject* args, PyObject *keywds)
2525
{
26-
const char *filename, *out;
26+
const char *filename;
27+
char *out;
2728
unsigned max_stack = 500, gc_min_objects = 1000, max_trace = 20;
2829
double gc_growth_trigger = 2;
2930
int debug_ast = 0, error;
3031
PyObject *env = NULL;
31-
struct JsonnetVM *vm;
32+
struct JsonnetVm *vm;
3233
static char *kwlist[] = {"filename", "max_stack", "gc_min_objects", "gc_growth_trigger", "env", "debug_ast", "max_trace", NULL};
3334

3435
(void) self;
@@ -67,25 +68,26 @@ static PyObject* evaluate_file(PyObject* self, PyObject* args, PyObject *keywds)
6768
out = jsonnet_evaluate_file(vm, filename, &error);
6869
if (error) {
6970
PyErr_SetString(PyExc_RuntimeError, out);
70-
jsonnet_cleanup_string(vm, out);
71+
jsonnet_realloc(vm, out, 0);
7172
jsonnet_destroy(vm);
7273
return NULL;
7374
} else {
7475
PyObject *ret = PyString_FromString(out);
75-
jsonnet_cleanup_string(vm, out);
76+
jsonnet_realloc(vm, out, 0);
7677
jsonnet_destroy(vm);
7778
return ret;
7879
}
7980
}
8081

8182
static PyObject* evaluate_snippet(PyObject* self, PyObject* args, PyObject *keywds)
8283
{
83-
const char *filename, *src, *out;
84+
const char *filename, *src;
85+
char *out;
8486
unsigned max_stack = 500, gc_min_objects = 1000, max_trace = 20;
8587
double gc_growth_trigger = 2;
8688
int debug_ast = 0, error;
8789
PyObject *env = NULL;
88-
struct JsonnetVM *vm;
90+
struct JsonnetVm *vm;
8991
static char *kwlist[] = {"filename", "src", "max_stack", "gc_min_objects", "gc_growth_trigger", "env", "debug_ast", "max_trace", NULL};
9092

9193
(void) self;
@@ -124,12 +126,12 @@ static PyObject* evaluate_snippet(PyObject* self, PyObject* args, PyObject *keyw
124126
out = jsonnet_evaluate_snippet(vm, filename, src, &error);
125127
if (error) {
126128
PyErr_SetString(PyExc_RuntimeError, out);
127-
jsonnet_cleanup_string(vm, out);
129+
jsonnet_realloc(vm, out, 0);
128130
jsonnet_destroy(vm);
129131
return NULL;
130132
} else {
131133
PyObject *ret = PyString_FromString(out);
132-
jsonnet_cleanup_string(vm, out);
134+
jsonnet_realloc(vm, out, 0);
133135
jsonnet_destroy(vm);
134136
return ret;
135137
}

‎case_studies/fractal/Makefile

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2014 Google Inc. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
ALL_IMAGES = $(shell cat service.list | grep '[.]packer.json$$')
16+
17+
default:
18+
../../jsonnet -J ../../lib -E USER --multi service.jsonnet > service.list
19+
$(MAKE) all_artifacts
20+
$(MAKE) terraform.done
21+
22+
all_artifacts: $(ALL_IMAGES:.packer.json=.packer.done)
23+
24+
%.packer.done: %.packer.json
25+
@echo "======== BUILDING IMAGE: $(basename $(basename $<)) ========"
26+
time packer build -force $< > $(basename $@).log 2>&1
27+
cp $< $@
28+
29+
terraform.done: terraform.tf
30+
@echo "======== MODIFYING DEPLOYMENT ========"
31+
terraform plan -out=terraform.plan
32+
@read -r -p "Are you sure? [y/N] " response && case "$$response" in [yY][eE][sS]|[yY]) terraform apply terraform.plan && cp $< $@ ;; esac
33+
34+
clean:
35+
rm -fv service.list *.packer.log *.packer.json *~ *.packer.done *.tf

‎case_studies/fractal/abstract_service.jsonnet

Lines changed: 298 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,298 @@
1+
/*
2+
Copyright 2014 Google Inc. All rights reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
local packer = import "packer.jsonnet";
18+
local terraform = import "terraform.jsonnet";
19+
local cassandra = import "cassandra.jsonnet";
20+
21+
{
22+
project:: error "You must override abstract_service and provide: project",
23+
bucket:: error "You must override abstract_service and provide: bucket",
24+
25+
cassandraRootPass:: error "You must override abstract_service and provide: cassandraRootPass",
26+
cassandraUserPass:: error "You must override abstract_service and provide: cassandraUserPass",
27+
cassandraUser:: "fractal",
28+
cassandraKeyspace:: "fractal",
29+
30+
cassandraNodes:: ["db1", "db2", "db3", "db4", "db5"],
31+
cassandraReplication:: "{ 'class' : 'SimpleStrategy', 'replication_factor' : 2 }",
32+
33+
CassandraConf:: cassandra.conf {
34+
cluster_name: "Fractal Cluster",
35+
rpc_address:: null, // Unset by making it hidden (::).
36+
listen_address:: null, // Unset by making it hidden (::).
37+
authenticator: "PasswordAuthenticator",
38+
seed_provider: [
39+
{
40+
class_name: "org.apache.cassandra.locator.SimpleSeedProvider",
41+
parameters: [ { seeds: std.join(", ", $.cassandraNodes) } ],
42+
},
43+
],
44+
},
45+
46+
cassandraInitCql:: [
47+
"CREATE USER %s WITH PASSWORD '%s';" % [$.cassandraUser, $.cassandraUserPass],
48+
local rep1 = "{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }";
49+
"CREATE KEYSPACE %s WITH REPLICATION = %s;" % [$.cassandraKeyspace, rep1],
50+
"USE %s;" % $.cassandraKeyspace,
51+
"CREATE TABLE discoveries("
52+
+ "Date TEXT, TimeId TIMEUUID, Text TEXT, X FLOAT, Y FLOAT, L INT, "
53+
+ "PRIMARY KEY(Date, TimeId));",
54+
"ALTER KEYSPACE %s WITH REPLICATION = %s;" % [$.cassandraKeyspace, $.cassandraReplication],
55+
"ALTER KEYSPACE system_auth WITH REPLICATION = %s;" % [$.cassandraReplication],
56+
],
57+
58+
ApplicationConf:: {
59+
width: 256,
60+
height: 256,
61+
thumb_width: 64,
62+
thumb_height: 64,
63+
iters: 200,
64+
database: $.cassandraKeyspace,
65+
},
66+
67+
68+
ImageMixin:: {
69+
project_id: $.project,
70+
account_file: "service_account_key.json",
71+
bucket_name: $.bucket,
72+
73+
// For debugging:
74+
local network_debug = ["traceroute", "lsof", "iptraf", "tcpdump", "host", "dnsutils"],
75+
aptPackages +: ["vim", "git", "psmisc", "screen", "strace" ] + network_debug,
76+
},
77+
78+
MyFlaskImage:: packer.GcpDebianNginxUwsgiFlaskImage + $.ImageMixin,
79+
80+
81+
"frontend.packer.json": $.MyFlaskImage {
82+
name: "frontend-v15",
83+
module: "main",
84+
pipPackages +: ["httplib2", "cassandra-driver", "blist"],
85+
uwsgiConf +: { lazy: "true" }, // cassandra-driver does not survive fork()
86+
provisioners +: [
87+
packer.File {
88+
source: "frontend",
89+
destination: "/tmp/",
90+
},
91+
packer.RootShell { inline: [
92+
"mv /tmp/frontend/* /var/www/",
93+
"chown -R www-data.www-data /var/www/*",
94+
] },
95+
],
96+
},
97+
98+
"cassandra-primed.packer.json": packer.GcpDebianCassandraPrimedImage + $.ImageMixin {
99+
name: "cassandra-primed-v4",
100+
rootPassword: $.cassandraRootPass,
101+
clusterName: $.CassandraConf.cluster_name,
102+
},
103+
104+
105+
"imgproc.packer.json": $.MyFlaskImage {
106+
name: "imgproc-v5",
107+
module: "mandelbrot_service",
108+
109+
aptPackages +: ["g++", "libpng-dev"],
110+
111+
provisioners +: [
112+
packer.File {
113+
source: "imgproc",
114+
destination: "/tmp/",
115+
},
116+
packer.RootShell { inline: [
117+
"mv /tmp/imgproc/* /var/www/",
118+
"chown -R www-data.www-data /var/www/*",
119+
] },
120+
packer.RootShell { inline: [
121+
"g++ -Wall -Wextra -ansi -pedantic -O3 -ffast-math -g "
122+
+ "/var/www/mandelbrot.cpp -lpng -o /var/www/mandelbrot",
123+
] },
124+
],
125+
}, // imgproc.packer.json
126+
127+
128+
"terraform.tf": {
129+
130+
provider: {
131+
google: {
132+
account_file: "service_account_key.json",
133+
client_secrets_file: "service_account_key.json",
134+
project: $.project,
135+
region: "us-central1",
136+
}
137+
},
138+
139+
resource: {
140+
local resource = self,
141+
142+
google_compute_address: {
143+
frontend: { name: "frontend" },
144+
imgproc: { name: "imgproc" },
145+
},
146+
147+
google_compute_http_health_check: {
148+
fractal: {
149+
name: "fractal",
150+
port: 80,
151+
},
152+
},
153+
154+
google_compute_target_pool: {
155+
frontend: {
156+
name: "frontend",
157+
health_checks: ["${google_compute_http_health_check.fractal.name}"],
158+
instances: [ "us-central1-f/frontend" + k for k in [1, 2, 3, 4, 5, 6] ],
159+
},
160+
imgproc: {
161+
name: "imgproc",
162+
health_checks: ["${google_compute_http_health_check.fractal.name}"],
163+
instances: [ "us-central1-f/imgproc" + k for k in [1, 2, 3, 4] ],
164+
},
165+
},
166+
167+
google_compute_forwarding_rule: {
168+
frontend: {
169+
ip_address: "${google_compute_address.frontend.address}",
170+
name: "frontend",
171+
target: "${google_compute_target_pool.frontend.self_link}",
172+
port_range: "80",
173+
},
174+
imgproc: {
175+
ip_address: "${google_compute_address.imgproc.address}",
176+
name: "imgproc",
177+
target: "${google_compute_target_pool.imgproc.self_link}",
178+
port_range: "80",
179+
}
180+
},
181+
182+
google_compute_network: {
183+
fractal: {
184+
name: "fractal",
185+
ipv4_range: "10.0.0.0/16",
186+
},
187+
},
188+
189+
google_compute_firewall: terraform.GcpFirewallSsh + terraform.GcpFirewallHttp
190+
+ cassandra.Firewall {
191+
network:: "${google_compute_network.fractal.name}",
192+
},
193+
194+
// TODO: load balancers & instance groups
195+
FractalInstance:: terraform.GcpInstance {
196+
network +: {source: "${google_compute_network.fractal.name}"},
197+
tags +: ["fractal"],
198+
scopes +: ["devstorage.full_control"],
199+
},
200+
201+
CassandraInstance:: self.FractalInstance {
202+
machine_type: "n1-standard-1",
203+
tags +: ["fractal-db", "cassandra-server"],
204+
},
205+
206+
google_compute_instance: {
207+
["frontend" + k]: resource.FractalInstance {
208+
name: "frontend" + k,
209+
image: "frontend-v1",
210+
conf:: $.ApplicationConf {
211+
database_name: $.cassandraKeyspace,
212+
database_user: $.cassandraUser,
213+
database_pass: $.cassandraUserPass,
214+
imgproc: "${google_compute_address.imgproc.address}",
215+
db_endpoints: $.cassandraNodes,
216+
},
217+
tags +: ["fractal-frontend", "http-server"],
218+
startup_script +: [self.addFile(self.conf, "/var/www/conf.json")],
219+
}
220+
for k in [1, 2, 3]
221+
222+
} + {
223+
// First node
224+
["db1"]: resource.CassandraInstance {
225+
name: "db1",
226+
image: "cassandra-v1",
227+
startup_script +: [
228+
// Wait for the misconfigured cassandra to start up.
229+
cassandra.waitForCqlsh("cassandra", $.cassandraRootPass, "localhost"),
230+
// Kill it.
231+
"/etc/init.d/cassandra stop",
232+
// Drop in the correct configuration.
233+
self.addFile($.CassandraConf, "/etc/cassandra/cassandra.yaml"),
234+
// Start it up again.
235+
"/etc/init.d/cassandra start",
236+
// Wait until it can be contacted.
237+
cassandra.waitForCqlsh("cassandra", $.cassandraRootPass, "$HOSTNAME"),
238+
// Set up users, empty tables, etc.
239+
local cql = std.lines($.cassandraInitCql);
240+
"echo %s | cqlsh -u cassandra -p %s $HOSTNAME"
241+
% [std.escapeStringBash(cql), $.cassandraRootPass],
242+
],
243+
},
244+
245+
} + {
246+
// Runners up, will wait for db1 and then form a cluster
247+
["db" + k]: resource.CassandraInstance {
248+
name: "db" + k,
249+
image: "cassandra-v1",
250+
startup_script +: [
251+
// Wait for the misconfigured cassandra to start up.
252+
cassandra.waitForCqlsh("cassandra", $.cassandraRootPass, "localhost"),
253+
// Kill it.
254+
"/etc/init.d/cassandra stop",
255+
// Clean up the mess it caused due to being misconfigured.
256+
"rm -rf /var/lib/cassandra/*",
257+
// Drop in the correct configuration.
258+
self.addFile($.CassandraConf, "/etc/cassandra/cassandra.yaml"),
259+
// Wait for db1 to be available.
260+
cassandra.waitForCqlsh("cassandra", $.cassandraRootPass, "db1"),
261+
// Start it up again.
262+
"/etc/init.d/cassandra start",
263+
],
264+
}
265+
for k in [2, 3]
266+
267+
/*
268+
} + {
269+
// Top-up nodes, don't run unless seeds are contactable
270+
["db" + k]: resource.CassandraInstance {
271+
name: "db" + k,
272+
image: "cassandra-primed-v4",
273+
startup_script +: [
274+
cassandra.waitForCqlsh("cassandra", $.cassandraRootPass, "localhost"),
275+
"/etc/init.d/cassandra stop",
276+
"rm -rf /var/lib/cassandra/*",
277+
self.addFile($.CassandraConf, "/etc/cassandra/cassandra.yaml"),
278+
"/etc/init.d/cassandra start",
279+
],
280+
}
281+
for k in [4, 5]
282+
*/
283+
284+
} + {
285+
["imgproc" + k]: resource.FractalInstance {
286+
name: "imgproc" + k,
287+
image: "imgproc-v1",
288+
tags +: ["fractal-imgproc", "http-server"],
289+
startup_script +: [self.addFile($.ApplicationConf, "/var/www/conf.json")],
290+
}
291+
for k in [1, 2, 3, 4]
292+
}
293+
294+
}
295+
296+
} // deployment.tf
297+
298+
}

0 commit comments

Comments
 (0)