-
-
Notifications
You must be signed in to change notification settings - Fork 66
/
build.py
executable file
·632 lines (578 loc) · 22.9 KB
/
build.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
#!/usr/bin/env python3
#
# `build.py` - Build the wheels and Docker containers needed for this
# application
# ===================================================================
#
# Imports
# -------
import locale
import os
import subprocess
import sys
from shutil import copyfile
import yaml
import toml
import pdb
from collections import OrderedDict
from datetime import datetime
# use python-dotenv >= 0.21.0
from dotenv import load_dotenv
from rich.console import Console
from rich.live import Live
from rich.table import Table
from sqlalchemy import create_engine, inspect
import rich
from rsptx.cl_utils.core import pushd, stream_command, subprocess_streamer
console = Console()
# Check environment variables
# ---------------------------
print("Checking your environment")
if not os.path.exists(".env"):
console.print(
"No .env file found. Please copy sample.env to .env and edit it.",
style="bold red",
)
exit(1)
if "--verbose" in sys.argv:
VERBOSE = True
else:
VERBOSE = False
if "--help" in sys.argv:
console.print(
"""Usage: build.py [--verbose] [--help] [--all] [--push]
--all build all containers, including author and worker
--push push all containers to a container registry
--one <service> build just one container, e.g. --one author
--restart restart the container(s) after building
--clean remove all containers and images before starting
--verbose show more output
--env check key environment variables and exit (verbose is T)
If something in the build does not work or you have questions about setup or environment
variables or installation, please check out our developer documentation.
https://runestone-monorepo.readthedocs.io/en/latest/developing.html
"""
)
exit(0)
if "--clean" in sys.argv:
console.print("Removing all containers and images...", style="bold")
ret = subprocess.run(["docker", "system", "prune", "--force"], capture_output=True)
if ret.returncode == 0:
console.print("All containers and images removed successfully", style="green")
else:
console.print("Failed to remove all containers and images", style="bold red")
exit(1)
# read the version from pyproject.toml
with open("pyproject.toml") as f:
pyproject = toml.load(f)
version = pyproject["tool"]["poetry"]["version"]
if "--push" in sys.argv:
try:
with open(".last_version") as f:
last_version = f.read().strip()
except FileNotFoundError:
with open(".last_version", "w") as f:
f.write(version)
last_version = version
if last_version == version:
update_version = input(
f"Do you want to update the version number ({version}) in pyproject.toml?"
)
if update_version.lower() in ["y", "yes"]:
new_version = input("Enter the new version number: ")
subprocess.run(
["poetry", "version", new_version], capture_output=True, check=True
)
console.out("Version updated, don't forget to commit the change.")
version = new_version
res = subprocess.run("docker info", shell=True, capture_output=True)
if res.returncode != 0:
console.print(
"Docker is not running. Please start it and try again.", style="bold red"
)
exit(1)
# make a fresh build.log for this build
with open("build.log", "w") as f:
f.write("")
if "--env" in sys.argv:
VERBOSE = True
console.print("Checking environment variables", style="bold")
# Per the [docs](https://pypi.org/project/python-dotenv/), load `.env` into
# environment variables.
load_dotenv()
table = Table(title="Environment Variables")
table.add_column("Variable", justify="right", style="grey62", no_wrap=True)
if VERBOSE:
table.add_column("Value", style="magenta")
else:
table.add_column("Set", style="magenta")
finish = False
for var in [
"RUNESTONE_PATH",
"RUNESTONE_HOST",
"SERVER_CONFIG",
"BOOK_PATH",
"WEB2PY_CONFIG",
"JWT_SECRET",
"WEB2PY_PRIVATE_KEY",
"COMPOSE_PROFILES",
"DBURL",
"DEV_DBURL",
]:
if var not in os.environ:
table.add_row(var, "[red]No[/red]")
finish = True
else:
if VERBOSE:
table.add_row(var, os.environ[var])
else:
table.add_row(var, "[green]Yes[/green]")
if "DC_DBURL" not in os.environ:
table.add_row("DC_DBURL", "[red]No[/red] will default to DBURL")
if "DBURL" not in os.environ:
table.add_row("DBURL", "[red]No[/red] please set it in .env")
finish = True
else:
if VERBOSE:
table.add_row("DC_DBURL", os.environ["DC_DBURL"])
else:
table.add_row("DC_DBURL", "[green]Yes[/green]")
if "DC_DEV_DBURL" not in os.environ:
table.add_row("DC_DEV_DBURL", "[red]No[/red] will default to DEV_DBURL")
if "DEV_DBURL" not in os.environ:
table.add_row("DEV_DBURL", "[red]No[/red] please set it in .env")
finish = True
else:
if VERBOSE:
table.add_row("DC_DEV_DBURL", os.environ["DC_DEV_DBURL"])
else:
table.add_row("DC_DEV_DBURL", "[green]Yes[/green]")
console.print(table)
cprofs = os.environ.get("COMPOSE_PROFILES", "")
if "basic" in cprofs:
console.print("Basic profile enabled")
if "2345" not in os.environ.get("DEV_DBURL", ""):
console.print(
"DEV_DBURL is not using port 2345. This is required to connect to a dockerized postgres instance. Please set it in .env",
style="bold red",
)
finish = True
else:
console.print(
"You are configured to run your own DB instance. Make sure it is running."
)
console.print("If you don't want to run your own, then enable the basic profile.")
if "2345" in os.environ.get("DEV_DBURL", ""):
console.print(
"DEV_DBURL is using port 2345. This is probably not correct if you are running postgres outside of docker",
style="bold red",
)
finish = True
if "COMPOSE_PROFILES" not in os.environ:
console.print(
"COMPOSE_PROFILES not set. Options include: basic, author, dev, production",
style="bold red",
)
console.print(
"""
The following services [red]will not be started[/red] unless you use set your COMPOSE_PROFILES in .env
[green]db, author, worker pgbouncer, nginx_dstart_dev[/green]
See the sample.env file for more information to set this variable.
You can also use the --profile switch with docker compose to enable a specific profile.
""",
style="bold",
)
finish = False
if "--env" in sys.argv:
exit(0)
if not os.path.isfile("bases/rsptx/web2py_server/applications/runestone/models/1.py"):
console.print("Copying 1.py.prototype to 1.py")
copyfile(
"bases/rsptx/web2py_server/applications/runestone/models/1.py.prototype",
"bases/rsptx/web2py_server/applications/runestone/models/1.py",
)
if finish:
console.print(
"Your environment is not set up correctly. Please define the environment variables listed above.",
style="bold red",
)
exit(1)
ym = yaml.load(open("docker-compose.yml"), yaml.FullLoader)
# remove the redis service from the list since we don't customize it
del ym["services"]["redis"]
# add the interactives service to the list so it gets built before the others,
# assignments depend on it. We do not build a container for it, but we do build the wheels
ym["services"]["interactives"] = {"build": {"context": "./projects/interactives"}}
ym["services"]["interactives"]["image"] = "skip"
ym["services"] = OrderedDict(ym["services"]) # convert to ordered dict
ym["services"].move_to_end("interactives", last=False)
if "--all" not in sys.argv:
# remove the author and worker services from the list since we don't customize them
del ym["services"]["author"]
del ym["services"]["worker"]
if "--one" in sys.argv:
svc_to_build = sys.argv[sys.argv.index("--one") + 1]
# remove all services except the one we want to build
for svc in list(ym["services"].keys()):
if svc != svc_to_build:
del ym["services"][svc]
# Attempt to determine the encoding of data returned from stdout/stderr of
# subprocesses. This is non-trivial. See the discussion at [Python's
# sys.stdout](https://docs.python.org/3/library/sys.html#sys.stdout). First, try
# `locale.getencoding()` (requires >= 3.11), with a fallback to
# `locale.getpreferredencoding()`.
#
# This is motivated by errors on Windows under Python 3.10:
#
# Traceback (most recent call last):
# File "C:\Users\bjones\Documents\git\rs\build.py", line 94, in <module>
# f.write(res.stdout.decode("utf-8"))
# File "C:\Users\bjones\AppData\Local\Programs\Python\Python310\lib\encodings\cp1252.py", line 19, in encode
# return codecs.charmap_encode(input,self.errors,encoding_table)[0]
# UnicodeEncodeError: 'charmap' codec can't encode characters in position 55319-55358: character maps to <undefined>
try:
# Since this isn't defined on all Python versions, tell mypy to ignore the
# following error.
stdout_err_encoding = locale.getencoding() # type: ignore[attr-defined]
except AttributeError:
stdout_err_encoding = locale.getpreferredencoding()
# Now initialize the build.log files for all services
for service in ym["services"]:
now = datetime.now()
try:
projdir = ym["services"][service]["build"]["context"]
except KeyError:
continue
with pushd(ym["services"][service]["build"]["context"]):
with open("build.log", "w") as f:
f.write(f"Build log for {service} on {now}\n")
def progress_wheel():
chars = "x+"
progress_wheel.counter += 1
return chars[progress_wheel.counter % 2]
progress_wheel.counter = 0
def generate_wheel_table(status: dict) -> Table:
table = Table(title="Build Python Wheels")
table.add_column("Wheel", justify="right", style="white", no_wrap=True)
table.add_column("Built", style="magenta")
for service in status:
table.add_row(service, status[service])
return table
if "--install" in sys.argv:
for proj in ym["services"].keys():
if (
"build" not in ym["services"][proj]
or ym["services"][proj]["build"]["context"] == "./"
):
continue
projdir = ym["services"][proj]["build"]["context"]
if os.path.isdir(projdir):
with pushd(projdir):
if os.path.isfile("pyproject.toml"):
console.print(f"Installing {proj}")
res = subprocess.run(
["poetry", "install", "--with=dev"], capture_output=True
)
if res.returncode == 0:
console.print(f"Installed {proj}")
else:
console.print(f"Failed to install {proj}")
if VERBOSE:
console.print(res.stderr.decode(stdout_err_encoding))
else:
with open("build.log", "a") as f:
f.write(res.stderr.decode(stdout_err_encoding))
else:
console.print(f"Skipping {proj} as it has no pyproject.toml")
continue
sys.exit(0)
# Build wheels
# ------------
status = {}
with Live(generate_wheel_table(status), refresh_per_second=4) as lt:
status = {}
for proj in ym["services"].keys():
if (
"build" not in ym["services"][proj]
or ym["services"][proj]["build"]["context"] == "./"
):
status[proj] = "[blue]Skipped[/blue]"
lt.update(generate_wheel_table(status))
continue
projdir = ym["services"][proj]["build"]["context"]
if os.path.isdir(projdir):
with pushd(projdir):
if os.path.isfile("build.py"):
status[proj] = "[grey62]pre build...[/grey62]"
lt.update(generate_wheel_table(status))
res = subprocess.run(
["python", "build.py", "--fromroot"], capture_output=True
)
if res.returncode == 0:
status[proj] = "[green]Yes[/green]"
lt.update(generate_wheel_table(status))
with open("build.log", "a") as f:
f.write(res.stdout.decode(stdout_err_encoding))
else:
status[proj] = f"[red]Fail[/red] See {projdir}/build.log"
lt.update(generate_wheel_table(status))
if VERBOSE:
console.print(res.stderr.decode(stdout_err_encoding))
else:
with open("build.log", "a") as f:
f.write(res.stderr.decode(stdout_err_encoding))
f.write(res.stdout.decode(stdout_err_encoding))
sys.exit(1)
if os.path.isfile("pyproject.toml"):
status[proj] = "[grey62]building...[/grey62]"
lt.update(generate_wheel_table(status))
toml_stat = os.stat("pyproject.toml")
lock_stat = os.stat("poetry.lock")
if toml_stat.st_mtime > lock_stat.st_mtime:
res = subprocess.run(
["poetry", "lock", "--no-update"], capture_output=True
)
if res.returncode != 0:
status[proj] = (
f"[red]Fail[/red] probable dependency conflict see {projdir}/build.log"
)
lt.update(generate_wheel_table(status))
if VERBOSE:
console.print(res.stderr.decode(stdout_err_encoding))
else:
with open("build.log", "a") as f:
f.write(res.stderr.decode(stdout_err_encoding))
sys.exit(1)
res = subprocess.run(
["poetry", "build-project"], capture_output=True
)
if res.returncode == 0:
status[proj] = "[green]Yes[/green]"
lt.update(generate_wheel_table(status))
with open("build.log", "a") as f:
f.write(res.stdout.decode(stdout_err_encoding))
else:
status[proj] = f"[red]Fail[/red] see {projdir}/build.log"
lt.update(generate_wheel_table(status))
if VERBOSE:
console.print(res.stderr.decode(stdout_err_encoding))
else:
with open("build.log", "a") as f:
f.write(res.stderr.decode(stdout_err_encoding))
sys.exit(1)
else:
status[proj] = "[blue]Skipped[/blue]"
lt.update(generate_wheel_table(status))
# Generate a table for the Live object
# see https://rich.readthedocs.io/en/stable/live.html?highlight=update#basic-usage
def generate_table(status: dict) -> Table:
table = Table(title="Build Docker Images")
table.add_column("Service", justify="right", style="white", no_wrap=True)
table.add_column("Built", style="magenta")
for service in status:
table.add_row(f"{service}", status[service])
return table
# Build Docker containers
# -----------------------
console.print(
"Building docker images (see build.log for detailed progress)...", style="bold"
)
status = {}
with Live(generate_table(status), refresh_per_second=4) as lt:
status = {}
for service in ym["services"]:
if service == "interactives":
status[service] = "[grey62]skipped...[/grey62]"
lt.update(generate_table(status))
continue
status[service] = "[grey62]building...[/grey62]"
lt.update(generate_table(status))
# to use a different wheel without editing Dockerfile use --build-arg wheel="wheelname"
command_list = [
"docker",
"compose",
"-f",
"docker-compose.yml",
"--progress",
"plain",
"build",
service,
]
with open("build.log", "ab") as f:
ret = subprocess.run(
command_list,
capture_output=True,
)
f.write(ret.stdout)
f.write(ret.stderr)
if ret.returncode == 0:
status[service] = "[green]Yes[/green]"
lt.update(generate_table(status))
else:
status[service] = "Failed"
lt.update(generate_table(status))
console.print(
f"There was an error building {service} see build.log for details",
style="bold red",
)
exit(1)
# Now if the --push flag was given, push the images to Docker Hub
# For this next part to work you need to be logged in to a docker hub account or the
# runestone private registry on digital ocean. The docker-compose.yml file has the
# image names set up to push to the runestone registry on digital ocean.
if "--push" in sys.argv:
console.print("Pushing docker images to Docker Hub...", style="bold")
for service in ym["services"]:
if "image" in ym["services"][service]:
image = ym["services"][service]["image"]
if "ghcr.io" not in image:
continue
console.print(f"Pushing {image}")
ret1 = subprocess.run(
["docker", "tag", image, f"{image}:v{version}"], check=True
)
# we do 2 pushes because the version tag is not automatically pushed
# and we want to push both the latest and the version tagged images
# but it is low cost as we only push the layers that are not already
# on the server.
ret2 = subprocess.run(["docker", "push", "--quiet", image], check=True)
ret3 = subprocess.run(
["docker", "push", "--quiet", f"{image}:v{version}"], check=True
)
if ret1.returncode + ret2.returncode + ret3.returncode == 0:
console.print(f"{image} pushed successfully")
ret = 0
else:
console.print(f"{image} failed to push", style="bold red")
exit(1)
console.print("Docker images pushed successfully", style="green")
with open(".last_version", "w") as f:
f.write(version)
# Check the database
# ------------------
engine = create_engine(os.environ["DEV_DBURL"])
try:
inspector = inspect(engine)
connection = engine.connect()
except Exception as e:
console.print(f"Failed to connect to the database: {e}", style="bold red")
exit(1)
DBOK = True
# Check if the alembic_version table is present in the database
try:
inspector.get_table_names().index("useinfo")
except:
console.print(
"It appears your database has not been initialized or your database is not started. Please see the documentation for how to initialize the database.",
style="bold red",
)
DBOK = False
try:
inspector.get_table_names().index("alembic_version")
except:
console.print(
"The Migrations have not been set up for your database. You need to run the alembic stamp head command to create it.",
style="bold red",
)
DBOK = False
if DBOK:
console.print("Database appears to be set up correctly", style="green")
console.print("Will now check for any needed migrations")
res = subprocess.run(["alembic", "heads"], check=True, capture_output=True)
head = res.stdout.decode("utf-8").split()[0].strip()
console.print(f"Alembic head is {head}")
res = subprocess.run(["alembic", "current"], check=True, capture_output=True)
current = res.stdout.decode("utf-8").split("\n")[-2].split()[0].strip()
console.print(f"Alembic current is {current}")
if res.returncode == 0:
if head == current:
console.print(
"Your database schema appears to be up to date", style="green"
)
else:
console.print("Database migrations are needed. Running them now...")
res = subprocess.run(["alembic", "upgrade", "head"], check=True)
if res.returncode == 0:
console.print("Migrations completed successfully", style="green")
else:
console.print("Migrations failed", style="bold red")
res = subprocess.run(
["alembic", "upgrade", "--sql", "head"], capture_output=True
)
console.print(
"Try running the following commands manually in psql to see what went wrong."
)
console.print(
"Then run [green]alembic stamp head[/green] to mark the migrations as complete."
)
console.print(res.stdout.decode("utf-8"))
if not DBOK:
exit(1)
if "--restart" in sys.argv:
if "--all" in sys.argv:
command_list = [
"docker",
"compose",
"--profile",
"author",
"-f",
"docker-compose.yml",
"stop",
]
console.print("Restarting all services...", style="bold")
elif "--one" in sys.argv:
command_list = [
"docker",
"compose",
"-f",
"docker-compose.yml",
"stop",
svc_to_build,
]
console.print(f"Restarting the {svc_to_build} service...", style="bold")
else:
command_list = ["docker", "compose", "stop"]
console.print("Restarting non-author services...", style="bold")
ret1 = subprocess.run(
command_list,
capture_output=True,
)
if "--all" in sys.argv:
command_list = [
"docker",
"compose",
"--profile",
"author",
"-f",
"docker-compose.yml",
"up",
"-d",
]
elif "--one" in sys.argv:
command_list = [
"docker",
"compose",
"--profile",
"author",
"--profile",
"basic",
"--profile",
"dev",
"--profile",
"production",
"-f",
"docker-compose.yml",
"up",
"-d",
svc_to_build,
]
else:
command_list = ["docker", "compose", "up", "-d"]
ret2 = subprocess.run(
command_list,
capture_output=True,
)
if ret1.returncode + ret2.returncode == 0:
console.print("Runestone service(s) restarted successfully", style="green")
else:
console.print("Runestone services failed to restart", style="bold red")