/
Makefile-docker
292 lines (234 loc) Β· 9.95 KB
/
Makefile-docker
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
export PYTHON_COMMAND=python3
# As we're using user-local installs inside the docker-container we need
# to be cautious about uprading pip and not confusing it with the
# globally installed version. This will take `$PYTHONUSERBASE` and `$PIP_USER`
# into account.
# See https://github.com/pypa/pip/issues/7205
export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
APP=src/olympia/
NUM_ADDONS=10
NUM_THEMES=$(NUM_ADDONS)
NPM_ARGS :=
ifneq ($(NPM_CONFIG_PREFIX),)
NPM_ARGS := --prefix $(NPM_CONFIG_PREFIX)
endif
ifneq ($(NPM_CACHE_DIR),)
NPM_ARGS := $(NPM_ARGS) --cache $(NPM_CACHE_DIR)
endif
ifneq ($(NPM_DEBUG),)
NPM_ARGS := $(NPM_ARGS) --loglevel verbose
endif
NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
STATIC_CSS := static/css/node_lib/
STATIC_JS := static/js/node_lib/
STATIC_JQUERY_UI := static/js/node_lib/ui/
NODE_LIBS_CSS := \
@claviska/jquery-minicolors/jquery.minicolors.css \
@claviska/jquery-minicolors/jquery.minicolors.png \
# NODE_LIBS_JS and NODE_LIBS_JQUERY_UI are referenced in settings.MINIFY_BUNDLES - keep both lists in sync
NODE_LIBS_JS := \
less/dist/less.js \
jquery/dist/jquery.js \
jquery.browser/dist/jquery.browser.js \
jquery.cookie/jquery.cookie.js \
@claviska/jquery-minicolors/jquery.minicolors.js \
jszip/dist/jszip.js \
timeago/jquery.timeago.js \
underscore/underscore.js \
netmask/lib/netmask.js \
NODE_LIBS_JQUERY_UI := \
jquery-ui/ui/data.js \
jquery-ui/ui/scroll-parent.js \
jquery-ui/ui/widget.js \
jquery-ui/ui/widgets/mouse.js \
jquery-ui/ui/widgets/sortable.js
REQUIRED_FILES := \
Makefile \
Makefile-os \
Makefile-docker \
version.json \
/deps/package.json \
/deps/package-lock.json \
/addons-server-docker-container \
.PHONY: help_redirect
help_redirect:
@$(MAKE) help --no-print-directory
.PHONY: check_debian_packages
check_debian_packages: ## check the existence of multiple debian packages
./scripts/check_debian_packages.sh
.PHONY: check_pip_packages
check_pip_packages: ## check the existence of multiple python packages
./scripts/check_pip_packages.sh
.PHONY: check_files
check_files: ## check the existence of multiple files
@for file in $(REQUIRED_FILES); do test -f "$$file" || (echo "$$file is missing." && exit 1); done
@echo "All required files are present."
.PHONY: check_olympia_user
check_olympia_user: ## check if the olympia user exists and is current user
@if [ "$$(id -u olympia)" != "$$(id -u)" ]; then echo "The current user is not the olympia user."; exit 1; fi
@echo "The current user is the olympia user."
.PHONY: check_django
check_django: ## check if the django app is configured properly
echo 'from olympia.lib.settings_base import *' > settings_local.py
DJANGO_SETTINGS_MODULE='settings_local' python3 ./manage.py check
rm settings_local.py
.PHONY: check
check: check_files check_olympia_user check_debian_packages check_pip_packages check_django
.PHONY: initialize_db
initialize_db: ## create a new database
rm -rf ./user-media/* ./tmp/*
$(PYTHON_COMMAND) manage.py create_db --force
$(PYTHON_COMMAND) manage.py migrate --noinput
$(PYTHON_COMMAND) manage.py loaddata initial.json
$(PYTHON_COMMAND) manage.py import_prod_versions
$(PYTHON_COMMAND) manage.py createsuperuser \
--no-input \
--username "$(SUPERUSER_USERNAME)" \
--email $(SUPERUSER_EMAIL)
$(PYTHON_COMMAND) manage.py loaddata zadmin/users
.PHONY: reindex_data
reindex_data: ## reindex the data in elasticsearch
$(PYTHON_COMMAND) manage.py reindex --force --noinput
.PHONY: populate_data
populate_data: ## populate a new database
# reindex --wipe will force the ES mapping to be re-installed. Useful to
# make sure the mapping is correct before adding a bunch of add-ons.
$(PYTHON_COMMAND) manage.py reindex --wipe --force --noinput
$(PYTHON_COMMAND) manage.py generate_addons --app firefox $(NUM_ADDONS)
$(PYTHON_COMMAND) manage.py generate_addons --app android $(NUM_ADDONS)
$(PYTHON_COMMAND) manage.py generate_themes $(NUM_THEMES)
# These add-ons are specifically useful for the addons-frontend
# homepage. You may have to re-run this, in case the data there
# changes.
$(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend
.PHONY: update_deps_pip
update_deps_pip: ## Install pip
# Work arounds "Multiple .dist-info directories" issue.
rm -rf /deps/build/*
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
.PHONY: update_deps_olympia
update_deps_olympia: ## Install the olympia local package
# pep 517 mode (the default) breaks editable install in our project. https://github.com/mozilla/addons-server/issues/16144
$(PIP_COMMAND) install --no-use-pep517 -e .
.PHONY: update_deps_development
update_deps_development: ## update the python and node dependencies for development
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
npm install $(NPM_ARGS) --no-save --include=dev
.PHONY: update_deps_production
update_deps_production: ## update the python and node dependencies for production
$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
npm ci $(NPM_ARGS) --include=prod
.PHONY: update_deps
update_deps: update_deps_pip update_deps_production update_deps_development update_deps_olympia## update the python and node dependencies
.PHONY: prune_deps
prune_deps: ## remove unused dependencies
npm prune $(NPM_ARGS) --production
.PHONY: update_db
update_db: ## run the database migrations
$(PYTHON_COMMAND) manage.py migrate --noinput
.PHONY: update_assets
update_assets:
# Copy files required in compress_assets to the static folder
mkdir -p $(STATIC_CSS) $(STATIC_JS) $(STATIC_JQUERY_UI)
for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done
for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done
for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done
# If changing this here, make sure to adapt tests in amo/test_commands.py
$(PYTHON_COMMAND) manage.py compress_assets
$(PYTHON_COMMAND) manage.py generate_jsi18n_files
# Collect static files: This MUST be run last or files will be missing
$(PYTHON_COMMAND) manage.py collectstatic --noinput
.PHONY: update
update: update_deps update_db update_assets ## update the dependencies, the database, and assets
.PHONY: reindex
reindex: ## reindex everything in elasticsearch, for AMO
$(PYTHON_COMMAND) manage.py reindex $(ARGS)
.PHONY: setup-ui-tests
setup-ui-tests:
rm -rf ./user-media/* ./tmp/*
# Reset the database and fake database migrations
$(PYTHON_COMMAND) manage.py create_db --force
$(PYTHON_COMMAND) manage.py migrate --noinput
# Reindex
$(PYTHON_COMMAND) manage.py reindex --force --noinput --wipe
# Let's load some initial data and import mozilla-product versions
$(PYTHON_COMMAND) manage.py loaddata initial.json
$(PYTHON_COMMAND) manage.py loaddata zadmin/users
$(PYTHON_COMMAND) manage.py loaddata src/olympia/access/fixtures/initial.json
$(PYTHON_COMMAND) manage.py import_prod_versions
# Create a proper superuser that can be used to access the API
$(PYTHON_COMMAND) manage.py waffle_switch super-create-accounts on --create
$(PYTHON_COMMAND) manage.py waffle_switch activate-autograph-signing on --create
$(PYTHON_COMMAND) manage.py generate_addons --app firefox $(NUM_ADDONS)
$(PYTHON_COMMAND) manage.py generate_addons --app android $(NUM_ADDONS)
$(PYTHON_COMMAND) manage.py generate_themes $(NUM_THEMES)
$(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend
# Now that addons have been generated, reindex.
$(PYTHON_COMMAND) manage.py reindex --force --noinput
.PHONY: perf-tests
perf-tests: setup-ui-tests
$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/perftests.txt
locust --no-web -c 1 -f tests/performance/locustfile.py --host "http://olympia.test"
.PHONY: lint
lint: ## lint the code
ruff check .
ruff format --check .
NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --check '**'
curlylint src/
lint-codestyle: lint
.PHONY: docs
docs: ## build the documentation
$(MAKE) -C docs html SPHINXOPTS='-nW'
.PHONY: debug
debug: ## connect for debugging
supervisorctl fg olympia
.PHONY: djshell
djshell: ## connect to django shell
$(PYTHON_COMMAND) ./manage.py shell_plus
.PHONY: dbshell
dbshell: ## connect to a database shell
$(PYTHON_COMMAND) ./manage.py dbshell
.PHONY: initialize
initialize: update_deps initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
.PHONY: reload
reload: ## force django code reload
uwsgi --reload ${HOME}/docker/artifacts/addons-server-uwsgi-master.pid
reload-uwsgi: reload
.PHONY: test
test: ## run the entire test suite
pytest $(APP) $(ARGS)
.PHONY: test_es
test_es: ## run the ES tests
pytest -m es_tests $(APP) $(ARGS)
.PHONY: test_no_es
test_no_es: ## run all but the ES tests
pytest -m "not es_tests" $(APP) $(ARGS)
.PHONY: test_force_db
test_force_db: ## run the entire test suite with a new database
pytest --create-db $(APP) $(ARGS)
.PHONY: tdd
tdd: ## run the entire test suite, but stop on the first error
pytest -x --pdb $(ARGS) $(APP)
.PHONY: test_failed
test_failed: ## rerun the failed tests from the previous run
pytest --lf $(ARGS) $(APP)
.PHONY: run_js_tests
run_js_tests: ## Run the JavaScript test suite (requires compiled/compressed assets).
NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest tests/js
.PHONY: watch_js_tests
watch_js_tests: ## Run+watch the JavaScript test suite (requires compiled/compressed assets).
NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- jest --watch
.PHONY: format
format: ## Autoformat our codebase.
NODE_PATH=$(NODE_MODULES) npm exec $(NPM_ARGS) -- prettier --write '**'
ruff check --fix-only .
ruff format .
.PHONY: extract_locales
extract_locales: ## extracts and merges translation strings
./scripts/run_l10n_extraction.sh
.PHONE: compile_locales
compile_locales: ## compiles translation strings
./locale/compile-mo.sh ./locale/
.PHONY: help_submake
help_submake:
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile-docker | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'