Skip to content

Add a Makefile to automate common command-line tasks #436

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Raruto wants to merge 9 commits into
base: dev
Choose a base branch
from
Draft

Add a Makefile to automate common command-line tasks #436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
608c7e5
add `Makefile` tasks
Raruto Jan 3, 2023
1d75e6f
make `clean` before `install`
Raruto Jan 4, 2023
d62b784
rename `Makefile.server.mk` into `Makefile.tasks.mk`
Raruto Jan 4, 2023
276a8df
remove `clean` before `install`
Raruto Jan 4, 2023
a7474db
use make tests within `test_runner.yml` workflow
Raruto Jan 4, 2023
cb83b08
convert some bash statements to posix shell
Raruto Jan 4, 2023
14fd69c
switch to legacy `docker-compose` binary on `test_runner.yml`
Raruto Jan 4, 2023
3aad6cc
fallback to `docker cp`
Raruto Jan 4, 2023
2a1b0d8
add `$${DOCKER_EXEC}` alias within `make tests`
Raruto Jan 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 5 additions & 81 deletions .github/workflows/test_runner.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,88 +21,12 @@ jobs:
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: extract_branch

- name: Starting Docker compose
run: docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml up -d

- name: Copying code into the container
run: |
docker cp ./ "$(docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml ps -q g3w-suite)":/code

- name: Installing Python requirements
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/ && python3 -m pip install pip==20.0.2 && pip3 install -r requirements_docker.txt && pip3 install -r requirements_huey.txt"

- name: Installing Caching Python requirements
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/ && pip3 install -r g3w-admin/caching/requirements.txt"

- name: Installing Filemanager Python requirements
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/ && pip3 install -r g3w-admin/filemanager/requirements.txt"

- name: Installing Qplotly Python requirements
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/ && pip3 install -r g3w-admin/qplotly/requirements.txt"

- name: Installing Openrouteservice Python requirements
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/ && pip3 install -r g3w-admin/openrouteservice/requirements.txt && pip3 install -r g3w-admin/openrouteservice/requirements_testing.txt"

- name: Building g3w-suite
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "/code/ci_scripts/build_suite.sh"

- name: Setting up g3w-suite
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "/code/ci_scripts/setup_suite.sh"

- name: Starting Django
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py runserver 0.0.0.0:8000" &

- name: Waiting for Django
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "wait-for-it -h localhost -p 8000 -t 240"

- name: Running g3w-admin core tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test core"

- name: Running g3w-admin qdjango tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test qdjango"

- name: Running g3w-admin usersmanage tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test usersmanage"

- name: Running g3w-admin client tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test client"

- name: Running g3w-admin editing tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test editing.tests"

- name: Running g3w-admin caching tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test caching"

- name: Running g3w-admin filemanager tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test filemanager"

- name: Running g3w-admin qplotly tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test qplotly"

- name: Running g3w-admin openrouteservice tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test openrouteservice"

- name: Running g3w-admin qtimeseries tests
- name: Running g3w-admin tests
run: |
docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml exec -T g3w-suite sh -c "cd /code/g3w-admin && python3 manage.py test qtimeseries"
make tests \
q=${{ matrix.qgis_version }} \
mode=test \
DOCKER_COMPOSE="docker-compose -f docker-compose.${{ matrix.qgis_version }}.yml"
Copy link
Copy Markdown
Contributor Author

@Raruto Raruto Jan 4, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently the automated tests are run on the Ubuntu 20.04 runner image:

g3w-admin/.github/workflows/test_runner.yml

Lines 7 to 9 in 10f48f8

ci_tests:
runs-on: ubuntu-20.04

As per actions/runner-images/images/linux/Ubuntu2004-Readme.md, the docker compose (v2) is installed, so why we can't just use that instead of docker-compose (v1)?

Docker Amazon ECR Credential Helper 0.6.0
Docker Compose v1 1.29.2
Docker Compose v2 2.14.0+azure-1
Docker-Buildx 0.9.1
Docker-Moby Client 20.10.21+azure-2
Docker-Moby Server 20.10.21+azure-2

Copy link
Copy Markdown
Contributor Author

@Raruto Raruto Jan 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

Docker Compose v1 1.29.2
Docker Compose v2 2.14.0+azure-1

This happens because some of these packets are available for GitHub Actions or Azure DevOps (that's why Docker Compose v2 2.14.0+azure-1 is only available for Azure V1 pipelines)

Ref: runner-images#faqs

runner-images/images/linux/scripts/installers/docker-compose.sh

#!/bin/bash -e
################################################################################
##  File:  docker-compose.sh
##  Desc:  Installs Docker Compose
################################################################################

# Install docker-compose v1 from releases
URL="https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64"
curl -L $URL -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose

invoke_tests "Tools" "Docker-compose v1"

runner-images/images/win/scripts/Installers/Install-Docker.ps1

Write-Host "Install-Package Docker-Compose v1"
Choco-Install -PackageName docker-compose

Write-Host "Install-Package Docker-Compose v2"
$dockerComposev2Url = "https://github.com/docker/compose/releases/latest/download/docker-compose-windows-x86_64.exe"
$cliPluginsDir = "C:\ProgramData\docker\cli-plugins"
New-Item -Path $cliPluginsDir -ItemType Directory
Start-DownloadWithRetry -Url $dockerComposev2Url -Name docker-compose.exe -DownloadPath $cliPluginsDir


- name: Setup tmate session
if: ${{ failure() }}
Expand Down
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,6 @@ run_local*.sh
/pymast

/shared-volume

Makefile.semver.mk
Makefile.venv.mk
6 changes: 5 additions & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
FROM g3wsuite/g3w-suite-deps:latest
LABEL maintainer="Gis3w" Description="This image is used to install python requirements and code for g3w-suite local testing" Vendor="Gis3w" Version="1.0"
LABEL \
maintainer="Gis3w" \
Description="This image is used to install python requirements and code for g3w-suite local testing" \
Vendor="Gis3w"\
Version="1.0"

# set env vars necessary to correctly install python3 requirements
ARG CPLUS_INCLUDE_PATH=/usr/include/gdal
Expand Down
53 changes: 53 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
##
# Makefile entrypoint
##

PROJECT_NAME = G3W-ADMIN

##
# see: <https://github.com/g3w-suite/makefiles/>
##
INCLUDE_MAKEFILES_RELEASE = v0.1.2
INCLUDE_MAKEFILES = Makefile.semver.mk Makefile.venv.mk

##
# Get a list of tasks when `make` is run without args.
##
default: list-tasks
help: list-tasks

##
# Download required packages
##
install: $(INCLUDE_MAKEFILES) venv install-reqs
@:

##
# Download and include any makefiles
# specified in the INCLUDE_MAKEFILES list
##
$(INCLUDE_MAKEFILES):
wget https://raw.githubusercontent.com/g3w-suite/makefiles/$(INCLUDE_MAKEFILES_RELEASE)/$@
$(foreach i, ${INCLUDE_MAKEFILES}, $(eval include $i))

##
# Remove downloaded makefiles
##
clean:
rm $(INCLUDE_MAKEFILES)

##
# Print a list of available tasks
#
# https://stackoverflow.com/a/26339924
##
list-tasks:
@echo Available tasks:
@echo ----------------
@$(MAKE) -pRrq -f $(firstword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | grep -E -v -e '^[^[:alnum:]]' -e '^$@$$'
@echo

##
# Include here any project specific tasks
##
include Makefile.tasks.mk
109 changes: 109 additions & 0 deletions Makefile.assets.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
##
# Makefile for G3W-ADMIN assets (CSS/JS, fonts, images)
##

##
# Common files and folder paths
##
BOWER_COMPONENTS_FOLDER = g3w-admin/core/static/bower_components
LOCAL_SETTINGS_FILE = g3w-admin/base/settings/local_settings.py
SHARED_VOLUME = /shared-volume
APPS_FOLDER = g3w-admin

##
# TODO: find out how to remove gulp as dependency (ref: gulp-useref, gulp-uglify, gulp-clean-css)
#
# Generate CSS an JS bundles:
#
# g3w-admin/templates/base.html --> g3w-admin/core/static/dist/css/vendor.min.css
# g3w-admin/templates/base.html --> g3w-admin/core/static/dist/css/g3wadmin.min.css
# g3w-admin/templates/base.html --> g3w-admin/core/static/dist/js/vendor.min.js
# g3w-admin/templates/base.html --> g3w-admin/core/static/dist/js/g3wadmin.min.js
#
##
build-assets: icheck_png fonts font-summernote
gulp build && rm g3w-admin/core/static/dist/base.html

##
# Update css folder:
#
# g3w-admin/core/static/dist/css/ <-- g3w-admin/core/static/bower_components/icheck/skins/flat/blue*.png
# g3w-admin/core/static/dist/css/ <-- g3w-admin/core/static/bower_components/icheck/skins/flat/green*.png
##
icheck_png: LOOKUP_RULE = -maxdepth 1 -type f \( -name "blue*.png" -or -name "green*.png" \)
icheck_png: INPUT_FILES = $(shell find g3w-admin/core/static/bower_components/icheck/skins/flat/ $(LOOKUP_RULE))
icheck_png: DEST_FOLDER = g3w-admin/core/static/dist/css/
icheck_png:
cp -vi ${INPUT_FILES} ${DEST_FOLDER}

##
# Update fonts folder:
#
# g3w-admin/core/static/dist/fonts/ <-- g3w-admin/core/static/bower_components/**/*.{eot,ttf,woff,woff2}
# g3w-admin/core/static/dist/fonts/ <-- g3w-admin/core/static/modules/**/*.{eot,ttf,woff,woff2}
##
fonts: LOOKUP_RULE = -type f \( -name "*.eot" -or -name "*.ttf" -or -name "*.woff" -or -name "*.woff2" \)
fonts: INPUT_FILES = $(shell find g3w-admin/core/static/bower_components $(LOOKUP_RULE)) $(shell find g3w-admin/core/static/modules/ $(LOOKUP_RULE))
fonts: DEST_FOLDER = g3w-admin/core/static/dist/fonts/
fonts:
cp -vi ${INPUT_FILES} ${DEST_FOLDER}

##
# Update fonts folder:
#
# g3w-admin/core/static/dist/css/font/ <-- g3w-admin/core/static/modules/summernote/font/*.{eot,ttf,woff,woff2}
##
fonts-summernote: LOOKUP_RULE = -maxdepth 1 -type f \( -name "*.eot" -or -name "*.ttf" -or -name "*.woff" -or -name "*.woff2" \)
fonts-summernote: INPUT_FILES = $(shell find g3w-admin/core/static/modules/summernote/font/ $(LOOKUP_RULE))
fonts-summernote: DEST_FOLDER = g3w-admin/core/static/dist/css/font/
fonts-summernote:
cp -vi ${INPUT_FILES} ${DEST_FOLDER}

##
# Symlink folders:
#
# node_modules/@bower_components <--> g3w-admin/core/static/bower_components
##
$(BOWER_COMPONENTS_FOLDER):
yarn --ignore-engines --ignore-scripts --prod ;\
nodejs -e "try { require('fs').symlinkSync(require('path').resolve('node_modules/@bower_components'), $(BOWER_COMPONENTS_FOLDER), 'junction') } catch (e) { console.log(e); }"

##
# Copy file:
#
# docker_settings.py --> g3w-admin/base/settings/local_settings.py
##
$(LOCAL_SETTINGS_FILE):
cp ./settings_docker.py ./g3w-admin/base/settings/local_settings.py

##
# Make folder:
#
# /shared-volume/media
##
$(SHARED_VOLUME)/media:
cd $(SHARED_VOLUME) && mkdir media

##
# Make folder:
#
# /shared-volume/static
##
$(SHARED_VOLUME)/static: $(SHARED_VOLUME)/media
cd $(SHARED_VOLUME) && ln -s media static

##
# Make folder:
#
# /shared-volume/media/projects
##
$(SHARED_VOLUME)/media/projects: $(SHARED_VOLUME)/media
cd $(SHARED_VOLUME)/media && mkdir projects

##
# Make folder:
#
# /shared-volume/project_data
##
$(SHARED_VOLUME)/project_data:
cd $(SHARED_VOLUME) && mkdir project_data
Loading