Gitlab runner as rez packages

WIP

Build our module

# ./package.py

name = "gitlab_runner"
version = "0.0.0"
timestamp = 0

def commands():
    alias('help', f'python {root}/gitlab_runner/cli.py -h')
    alias('run', f'python {root}/gitlab_runner/cli.py run')
    alias('configure', f'python {root}/gitlab_runner/cli.py configure')
    alias('clear', f'python {root}/gitlab_runner/cli.py clear')

# ./gitlab_runner/consts.py

# Python built-in modules import
import configparser
import os
import platform
import socket
from pathlib import Path

# create dict to switch to network drive by platform
SMB_ROOT_DICT = {
    'Windows': Path('P:/'),
    'Linux': Path('/')
    'Darwin': Path('/')
}
INI_FILE_PATH = SMB_ROOT_DICT[platform.system()] / 'path' / 'to' / 'keys' / 'keys.ini'

config = configparser.ConfigParser()
config.read(INI_FILE_PATH)

class CONST_ENVIRONMENT(dict):
    ROOT_GITLAB_RUNNER = os.environ['ROOT_GITLAB_RUNNER']
    NAME_GITLAB_RUNNER = socket.gethostname()
    OS_NAME = platform.system()

class CONST_PATH(dict):
    CONFIG_FILE_PATH = Path.home() / 'gitlab_runner' / 'config.toml'
    BIN_GITLAB_RUNNER = Path(CONST_ENVIRONMENT.ROOT_GITLAB_RUNNER) / 'gitlab-runner'

class CONST_AUTH(dict):
    URL = config['gitlab_runner']['url']
    TOKEN = config['gitlab_runner']['key']

# ./gitlab_runner/gitlab_runner.py

# Python built-in modules import
import os

# local import
from gitlab_runner.consts import CONST_AUTH
from gitlab_runner.consts import CONST_ENVIRONMENT
from gitlab_runner.consts import CONST_PATH


def run(debug: bool = False):
    """Run gitlab_runner command.

    :param debug: Enable debug mode to get more log info, False by default.
    :type debug: bool, optional.
    """

    # Ensure gitlab_runner is configured
    configure()

    run_command = f"{CONST_PATH.BIN_GITLAB_RUNNER} {'--debug' if debug else ''} run  -c {CONST_PATH.CONFIG_FILE_PATH}"
    os.system(run_command)


def configure() -> None:
    """Configure gitlab_runner command."""

    if os.path.exists(CONST_PATH.CONFIG_FILE_PATH):
        return

    runner_register_command =f'\
         {CONST_PATH.BIN_GITLAB_RUNNER} register \
         --config {CONST_PATH.CONFIG_FILE_PATH} \
         --non-interactive \
         --url {CONST_AUTH.URL} \
         --registration-token {CONST_AUTH.TOKEN} \
         --name {CONST_ENVIRONMENT.NAME_GITLAB_RUNNER} \
         --tag-list shell,{CONST_ENVIRONMENT.OS_NAME} \
         --run-untagged \ # if you wand to run untagged jobs
         --executor shell'
    os.system(runner_register_command)


def clear():
    """Clear gitlab_runner config dir."""
    parent_dir = CONST_PATH.CONFIG_FILE_PATH.parent

    files_to_remove = [
        '.runner_system_id',
        'config.toml'
    ]

    for file_name in files_to_remove:
        file_path = parent_dir / file_name
        if not os.path.exists(file_path):
            continue
        os.remove(file_path)

# ./gitlab_runner/cli.py


# Python built-in modules import
import argparse

# local import
from gitlab_runner import gitlab_runner


class GitlabRunnerCLI(object):
    @classmethod
    def init_parser(cls) -> None:
        """Init the parser for the cli app."""

        parser = argparse.ArgumentParser(description='Gitlab runner wrapper')
        subparser = parser.add_subparsers(dest='command')

        # run
        run = subparser.add_parser('run', help='Start the Gitlab runner')
        run.add_argument(
            '--debug',
            type=bool,
            help='Enable debug mode to get more information',
            default=False,
            required=False,
        )

        # Configure
        subparser.add_parser('configure', help='Configure the Gitlab runner')

        # Clear
        subparser.add_parser('clear', help='Clear the Gitlab runner configuration')

        # Get args
        args = parser.parse_args()

        # Filter to only keep args without 'command'
        real_param_dict = {k: v for k, v in args.__dict__.items() if k != 'command'}

        return getattr(cls(), args.command)(**real_param_dict)

    def run(self, debug: bool) -> None:
        """Start the Gitlab runner.

        :param debug: Enable debug mode to get more information
        :type debug: bool
        """
        gitlab_runner.run(debug=debug)

    def configure(self) -> None:
        """Configure the Gitlab runner."""

        gitlab_runner.configure()

    def clear(self) -> None:
        """Clear the Gitlab runner configuration."""
        gitlab_runner.clear()


if __name__ == '__main__':
    GitlabRunnerCLI.init_parser()

Usage

Help command

rez env sq_gitlab_runner -- help # global help
rez env sq_gitlab_runner -- run -h # help for run command
rez env sq_gitlab_runner -- clear -h # help for clear command
rez env sq_gitlab_runner -- configure -h # help for configure command

Run commands:

rez env sq_gitlab_runner -- run # run gitlab-runner and register it if not already registered.
rez env sq_gitlab_runner -- run --debug True # Same as above but with debug log mode enabled.
rez env sq_gitlab_runner -- clear # clear gitlab-runner local configuration.
rez env sq_gitlab_runner -- configure # configure and register gitlab-runner.

Edit config

Configuration of the Gitlab-runner are created with the register command. You can found the configuration documentation here.

We are using the custom executor to manage on our side the custom clean up of the environment. That way we can use custom shell for custom-run-exec and custom-cleanup-exec.

You can get the complete list of argument by running gitlab-runner -h.

How the registration works ?

To register the gitlab-runner we are using the gitlab-runner register command. You can found the documentation here.

But in few words, we are using the token and the url of the gitlab instance to register the runner as a shared runner.

How jobs are executed ?

Jobs are executed in a custom shell. We have configured powershell as shell for windows and bash for linux and macos.

When a job is received, the gitlab runner will clone the repository in the build directory and then execute the job command.

When the job is finished, the gitlab runner will execute the cleanup command.

How the cleanup works ?

For the cleanup we are using the custom-cleanup-exec to execute a specific shell and command per platform.

For Windows we are using powershell and for linux and macos we are using rm as shell.

So in a few words, we are using the custom-cleanup-exec to execute a command that will delete the build directory.