From c4257b7233a95969f532d8e68346c1f6ef39e776 Mon Sep 17 00:00:00 2001 From: Anton Shuvalov Date: Tue, 16 Jul 2024 01:14:22 +0700 Subject: [PATCH 1/2] Add `cwd` to Job, JobConfig, and popen calls --- config.example.yaml | 1 + pyproject.toml | 2 +- zapusk/client/__main__.py | 2 + zapusk/client/api_client.py | 1 + zapusk/client/command.py | 16 +++++ zapusk/client/command_exec.py | 8 ++- zapusk/client/command_exec_test.py | 30 +++++++-- zapusk/client/command_list.py | 4 +- zapusk/models/job.py | 7 +++ zapusk/models/job_config.py | 8 ++- zapusk/server/controller_config_test.py | 50 +++------------ zapusk/server/controller_jobs.py | 3 + zapusk/server/controller_jobs_test.py | 57 ++++++----------- .../server/controller_scheduled_jobs_test.py | 61 +++++-------------- zapusk/server/controller_testcase.py | 58 ++++++++++++++++++ zapusk/services/config/config_parser_test.py | 9 +++ zapusk/services/config/service.py | 13 ++-- zapusk/services/config/service_test.py | 37 ++++++++++- .../backends/kawka/args_consumer.py | 4 ++ .../backends/kawka/consumer_test.py | 9 +++ .../backends/kawka/executor.py | 7 +++ .../backends/kawka/executor_test.py | 32 ++++++++++ .../scheduler_service/service_test.py | 8 +++ 23 files changed, 286 insertions(+), 141 deletions(-) create mode 100644 zapusk/server/controller_testcase.py diff --git a/config.example.yaml b/config.example.yaml index f4778af..a502f1a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,6 +14,7 @@ jobs: - name: Sleep 10 Seconds id: sleep_10 command: sleep 10 + cwd: /var/ - name: Sleep 30 Seconds group: parallel diff --git a/pyproject.toml b/pyproject.toml index 64d4046..f1e9ac7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "zapusk" -version = "0.1.0" +version = "0.1.1" description = "" authors = ["Anton Shuvalov "] readme = "README.md" diff --git a/zapusk/client/__main__.py b/zapusk/client/__main__.py index 8cf790c..e485472 100644 --- a/zapusk/client/__main__.py +++ b/zapusk/client/__main__.py @@ -1,3 +1,4 @@ +import os from type_docopt import docopt import importlib.metadata @@ -95,6 +96,7 @@ def main(): name=str(args["--name"]) if args["--name"] else None, schedule=str(args["--schedule"]) if args["--schedule"] else None, tail=bool(args["--tail"]), + cwd=os.getcwd(), ) return diff --git a/zapusk/client/api_client.py b/zapusk/client/api_client.py index 1fcc775..d475edc 100644 --- a/zapusk/client/api_client.py +++ b/zapusk/client/api_client.py @@ -27,6 +27,7 @@ class JobCreateFromConfigPayload(TypedDict): class JobCreateFromCommandPayload(TypedDict): command: str + cwd: str name: NotRequired[Optional[str]] group_id: NotRequired[Optional[str]] diff --git a/zapusk/client/command.py b/zapusk/client/command.py index c376159..5a2f08f 100644 --- a/zapusk/client/command.py +++ b/zapusk/client/command.py @@ -1,5 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING +from requests.exceptions import ConnectionError from .api_client import ApiClient, ApiClientError from .output import Output @@ -28,3 +29,18 @@ def print_json(self, json_data, one_line=False): def print_error(self, exception): self.output.error(exception, colors=self.colors) + + def handle_error(self, ex): + if type(ex) ==ApiClientError: + self.print_error(ex) + return + + if type(ex) == ConnectionError: + if "Connection refused by Responses" not in str(ex): + self.print_error(ApiClientError("Can not connect to the server. Please start server with `zapusk-server`")) + return + + raise ex + + + diff --git a/zapusk/client/command_exec.py b/zapusk/client/command_exec.py index 1d1a803..677eb5b 100644 --- a/zapusk/client/command_exec.py +++ b/zapusk/client/command_exec.py @@ -1,3 +1,4 @@ +import os from typing import Optional from zapusk.client.api_client import ApiClientError @@ -9,6 +10,7 @@ class CommandExec(Command): def run( self, command: str, + cwd: str, name: Optional[str] = None, group_id: Optional[str] = None, schedule: Optional[str] = None, @@ -23,6 +25,7 @@ def run( "group_id": group_id, "name": name, "schedule": schedule, + "cwd": cwd, } ) @@ -35,6 +38,7 @@ def run( "command": command, "group_id": group_id, "name": name, + "cwd": cwd, } ) @@ -44,5 +48,5 @@ def run( self.print_json(created_job) - except ApiClientError as ex: - self.print_error(ex) + except Exception as ex: + self.handle_error(ex) diff --git a/zapusk/client/command_exec_test.py b/zapusk/client/command_exec_test.py index da831d6..dcfa37c 100644 --- a/zapusk/client/command_exec_test.py +++ b/zapusk/client/command_exec_test.py @@ -1,4 +1,5 @@ import json +import os from unittest.mock import call, patch import responses from responses import matchers @@ -7,6 +8,7 @@ class TestCommandExec(CommandTestCase): + @responses.activate def test_should_exec_job(self): responses.post( @@ -19,6 +21,7 @@ def test_should_exec_job(self): "command": "echo 1", "group_id": "echo", "name": "Echo", + "cwd": "/home/anton/", } ) ], @@ -28,6 +31,7 @@ def test_should_exec_job(self): command="echo 1", group_id="echo", name="Echo", + cwd="/home/anton/", ) json_data = json.loads(self.printer.print.call_args[0][0]) @@ -45,6 +49,7 @@ def test_should_exec_scheduled_job(self): "command": "echo 1", "group_id": "echo", "name": "Echo", + "cwd": "/home/anton/", "schedule": "*/1 * * * *", } ) @@ -52,7 +57,11 @@ def test_should_exec_scheduled_job(self): ) self.command_manager.exec.run( - command="echo 1", group_id="echo", name="Echo", schedule="*/1 * * * *" + command="echo 1", + group_id="echo", + name="Echo", + schedule="*/1 * * * *", + cwd="/home/anton/", ) json_data = json.loads(self.printer.print.call_args[0][0]) @@ -66,7 +75,10 @@ def test_should_handle_error(self): json={"error": "ERROR"}, ) - self.command_manager.exec.run(command="echo 1") + self.command_manager.exec.run( + command="echo 1", + cwd="/home/anton/", + ) args = self.printer.print.call_args[0] message = json.loads(args[0]) @@ -82,13 +94,23 @@ def test_should_tail_job(self): responses.get( "http://example.com/jobs/1", status=200, - json={"data": {"id": 1, "log": "/var/tail.log"}}, + json={ + "data": { + "id": 1, + "log": "/var/tail.log", + "cwd": "/home/anton/", + }, + }, ) with patch( "zapusk.client.command_tail.tail", return_value=["log line 1", "log line 2"] ): - self.command_manager.exec.run(command="echo 1", tail=True) + self.command_manager.exec.run( + command="echo 1", + tail=True, + cwd="/home/anton/", + ) log_line1 = self.printer.print.call_args_list[0] log_line2 = self.printer.print.call_args_list[1] diff --git a/zapusk/client/command_list.py b/zapusk/client/command_list.py index 649a313..d60b773 100644 --- a/zapusk/client/command_list.py +++ b/zapusk/client/command_list.py @@ -25,5 +25,5 @@ def run( self.print_json(jobs) return - except ApiClientError as ex: - self.print_error(ex) + except Exception as ex: + self.handle_error(ex) diff --git a/zapusk/models/job.py b/zapusk/models/job.py index b17dd8a..00dfc88 100644 --- a/zapusk/models/job.py +++ b/zapusk/models/job.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum +import os from typing import Optional from .id_field import IdField @@ -64,6 +65,7 @@ def from_config(group_config: JobGroup, config: JobConfig): name=config.name, on_finish=config.on_finish, on_fail=config.on_fail, + cwd=config.cwd, ) group_config: JobGroup @@ -86,6 +88,11 @@ def from_config(group_config: JobGroup, config: JobConfig): job_group id """ + cwd: str = field(default_factory=lambda: os.environ["HOME"]) + """ + current working dir + """ + job_config_id: Optional[str] = None """ job_config id diff --git a/zapusk/models/job_config.py b/zapusk/models/job_config.py index 28aebc0..3294ce4 100644 --- a/zapusk/models/job_config.py +++ b/zapusk/models/job_config.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field +import os from typing import Optional from .base_model import BaseModel @@ -21,6 +22,11 @@ class JobConfig(BaseModel): shell command for the job """ + cwd: str = field(default_factory=lambda: os.environ["HOME"]) + """ + current working dir + """ + group: str = "default" """ Group id to run job in diff --git a/zapusk/server/controller_config_test.py b/zapusk/server/controller_config_test.py index 1d13c27..9b0fd0f 100644 --- a/zapusk/server/controller_config_test.py +++ b/zapusk/server/controller_config_test.py @@ -1,16 +1,7 @@ -from unittest import TestCase +import json -from flask import json -from testfixtures import TempDirectory +from .controller_testcase import ControllerTestCase -from zapusk.services import ( - ConfigService, - SchedulerService, - ExecutorManagerService, - ExecutorManagerKawkaBackend, -) - -from .api import create_app CONFIG_DATA = """ job_groups: @@ -25,43 +16,16 @@ - id: test1 name: Test1 command: test1 + cwd: /home/ - id: test2 name: Test2 command: test2 """ -class TestConfigController(TestCase): - def setUp(self) -> None: - self.temp_dir = TempDirectory() - config_file = self.temp_dir / "config.yml" - config_file.write_text(CONFIG_DATA) - - self.executor_manager_service = ExecutorManagerService( - backend=ExecutorManagerKawkaBackend(), - ) - self.config_service = ConfigService( - config_path=f"{self.temp_dir.path}/config.yml" - ) - self.scheduler_service = SchedulerService( - config_service=self.config_service, - executor_manager_service=self.executor_manager_service, - ) - self.scheduler_service.start() - - self.app = create_app( - executor_manager_service=self.executor_manager_service, - config_service=self.config_service, - scheduler_service=self.scheduler_service, - ) - self.test_client = self.app.test_client() - - def tearDown(self) -> None: - self.executor_manager_service.terminate() - self.scheduler_service.terminate() - self.temp_dir.cleanup() - +class TestConfigController(ControllerTestCase): def test_config_groups_list(self): + self.write_config(CONFIG_DATA) res = self.test_client.get("/config/groups/") data = json.loads(res.data) self.assertEqual( @@ -91,6 +55,8 @@ def test_config_groups_list(self): ) def test_config_jobs_list(self): + self.write_config(CONFIG_DATA) + self.replace_in_environ("HOME", "/home/kanye") res = self.test_client.get("/config/jobs/") data = json.loads(res.data) self.assertEqual( @@ -106,6 +72,7 @@ def test_config_jobs_list(self): "on_fail": None, "on_finish": None, "schedule": None, + "cwd": "/home/", }, { "args_command": None, @@ -116,6 +83,7 @@ def test_config_jobs_list(self): "on_fail": None, "on_finish": None, "schedule": None, + "cwd": "/home/kanye", }, ] }, diff --git a/zapusk/server/controller_jobs.py b/zapusk/server/controller_jobs.py index 406503d..971c641 100644 --- a/zapusk/server/controller_jobs.py +++ b/zapusk/server/controller_jobs.py @@ -1,3 +1,4 @@ +import os from flask import Blueprint, Response, abort, request from zapusk.lib.json_serdes import JsonSerdes from zapusk.models import Job, JobConfig, IdField @@ -27,6 +28,7 @@ def job_add(): body = request.json or {} job_config_id = body.get("job_config_id", None) + cwd = body.get("cwd", os.environ["HOME"]) # if no config id, let's try to execute it as a command if not job_config_id: @@ -59,6 +61,7 @@ def job_add(): id=cmd_id, name=name or f"{job_group.id}.{cmd_id}", command=command, + cwd=cwd, ), ) executor_manager_service.add(job_item) diff --git a/zapusk/server/controller_jobs_test.py b/zapusk/server/controller_jobs_test.py index 1b6d2a9..eb7c193 100644 --- a/zapusk/server/controller_jobs_test.py +++ b/zapusk/server/controller_jobs_test.py @@ -4,6 +4,7 @@ from testfixtures import TempDirectory +from zapusk.server.controller_testcase import ControllerTestCase from zapusk.services import ( ConfigService, SchedulerService, @@ -27,37 +28,12 @@ """ -class TestJobController(TestCase): - def setUp(self) -> None: - self.temp_dir = TempDirectory() - self.config_file = self.temp_dir / "config.yml" - self.config_file.write_text(CONFIG_DATA) +class TestJobController(ControllerTestCase): + def before_create_services(self): + self.write_config(CONFIG_DATA) + self.replace_in_environ("HOME", self.temp_dir.path) - self.executor_manager_service = ExecutorManagerService( - backend=ExecutorManagerKawkaBackend(), - ) - self.config_service = ConfigService( - config_path=f"{self.temp_dir.path}/config.yml" - ) - self.scheduler_service = SchedulerService( - config_service=self.config_service, - executor_manager_service=self.executor_manager_service, - ) - self.scheduler_service.start() - - self.app = create_app( - executor_manager_service=self.executor_manager_service, - config_service=self.config_service, - scheduler_service=self.scheduler_service, - ) - self.test_client = self.app.test_client() - - def tearDown(self) -> None: - self.executor_manager_service.terminate() - self.scheduler_service.terminate() - self.temp_dir.cleanup() - - def test_create_job(self): + def test_controller_jobs_create_job(self): res = self.test_client.post("/jobs/", json={"job_config_id": "echo"}) data = json.loads(res.data) @@ -68,6 +44,7 @@ def test_create_job(self): "args": [], "args_command": None, "command": "echo 1", + "cwd": self.temp_dir.path, "consumed_by": None, "created_at": ANY, "exit_code": None, @@ -91,7 +68,7 @@ def test_create_job(self): }, ) - def test_create_command(self): + def test_controller_jobs_create_command(self): res = self.test_client.post( "/jobs/", json={ @@ -109,6 +86,7 @@ def test_create_command(self): "args": [], "args_command": None, "command": "echo 42", + "cwd": self.temp_dir.path, "consumed_by": None, "created_at": ANY, "exit_code": None, @@ -132,7 +110,7 @@ def test_create_command(self): }, ) - def test_get_job(self): + def test_controller_jobs_get_job(self): res = self.test_client.post("/jobs/", json={"job_config_id": "echo"}) data = json.loads(res.data) @@ -147,6 +125,7 @@ def test_get_job(self): "args": [], "args_command": None, "command": "echo 1", + "cwd": self.temp_dir.path, "consumed_by": None, "created_at": ANY, "exit_code": None, @@ -170,7 +149,7 @@ def test_get_job(self): }, ) - def test_list_job(self): + def test_controller_jobs_list_job(self): res = self.test_client.post("/jobs/", json={"job_config_id": "echo"}) data = json.loads(res.data) @@ -186,6 +165,7 @@ def test_list_job(self): "args": [], "args_command": None, "command": "echo 1", + "cwd": self.temp_dir.path, "consumed_by": None, "created_at": ANY, "exit_code": None, @@ -210,7 +190,7 @@ def test_list_job(self): }, ) - def test_cancel_job(self): + def test_controller_jobs_cancel_job(self): res = self.test_client.post( "/jobs/", json={"command": "sleep 60", "name": "test_command"} ) @@ -227,6 +207,7 @@ def test_cancel_job(self): "args": [], "args_command": None, "command": "sleep 60", + "cwd": self.temp_dir.path, "consumed_by": ANY, "created_at": ANY, "exit_code": None, @@ -250,7 +231,7 @@ def test_cancel_job(self): }, ) - def test_get_unknown(self): + def test_controller_jobs_get_unknown(self): res = self.test_client.get(f"/jobs/420") data = json.loads(res.data) @@ -266,7 +247,7 @@ def test_create_without_body(self): data, {"error": "Request body contains no `command` or `job_config_id`"} ) - def test_create_with_unknown_jobgroup(self): + def test_controller_jobs_create_with_unknown_jobgroup(self): res = self.test_client.post( f"/jobs/", json={ @@ -279,7 +260,7 @@ def test_create_with_unknown_jobgroup(self): self.assertEqual(res.status, "404 NOT FOUND") self.assertEqual(data, {"error": 'group_id "unknown" not found'}) - def test_create_with_unknown_jobconfig_id(self): + def test_controller_jobs_create_with_unknown_jobconfig_id(self): res = self.test_client.post( f"/jobs/", json={ @@ -291,7 +272,7 @@ def test_create_with_unknown_jobconfig_id(self): self.assertEqual(res.status, "404 NOT FOUND") self.assertEqual(data, {"error": "Job with id `unknown` not found"}) - def test_cancel_unknown_job(self): + def test_controller_jobs_cancel_unknown_job(self): res = self.test_client.delete("/jobs/420") data = json.loads(res.data) diff --git a/zapusk/server/controller_scheduled_jobs_test.py b/zapusk/server/controller_scheduled_jobs_test.py index 94e2830..7d12ce4 100644 --- a/zapusk/server/controller_scheduled_jobs_test.py +++ b/zapusk/server/controller_scheduled_jobs_test.py @@ -1,17 +1,7 @@ import json -from unittest import TestCase from unittest.mock import ANY, patch -from testfixtures import TempDirectory - -from zapusk.services import ( - ConfigService, - SchedulerService, - ExecutorManagerService, - ExecutorManagerKawkaBackend, -) - -from .api import create_app +from .controller_testcase import ControllerTestCase CONFIG_DATA = """ jobs: @@ -22,37 +12,12 @@ """ -class TestJobController(TestCase): - def setUp(self) -> None: - self.temp_dir = TempDirectory() - self.config_file = self.temp_dir / "config.yml" - self.config_file.write_text(CONFIG_DATA) - - self.executor_manager_service = ExecutorManagerService( - backend=ExecutorManagerKawkaBackend(), - ) - self.config_service = ConfigService( - config_path=f"{self.temp_dir.path}/config.yml" - ) - self.scheduler_service = SchedulerService( - config_service=self.config_service, - executor_manager_service=self.executor_manager_service, - ) - self.scheduler_service.start() - - self.app = create_app( - executor_manager_service=self.executor_manager_service, - config_service=self.config_service, - scheduler_service=self.scheduler_service, - ) - self.test_client = self.app.test_client() - - def tearDown(self) -> None: - self.executor_manager_service.terminate() - self.scheduler_service.terminate() - self.temp_dir.cleanup() +class TestSchedulerJobController(ControllerTestCase): + def before_create_services(self): + self.write_config(CONFIG_DATA) + self.replace_in_environ("HOME", self.temp_dir.path) - def test_list(self): + def test_controller_scheduled_jobs_list(self): res = self.test_client.get("/scheduled-jobs/") data = json.loads(res.data) @@ -63,6 +28,7 @@ def test_list(self): { "args_command": None, "command": "echo 1", + "cwd": self.temp_dir.path, "group": "default", "id": "scheduled_echo", "name": "Echo", @@ -74,7 +40,7 @@ def test_list(self): }, ) - def test_create(self): + def test_controller_scheduled_jobs_create(self): res = self.test_client.post( "/scheduled-jobs/", json={ @@ -90,6 +56,7 @@ def test_create(self): { "data": { "args_command": None, + "cwd": self.temp_dir.path, "command": "echo 42", "group": "default", "id": "scheduled.1", @@ -101,7 +68,7 @@ def test_create(self): }, ) - def test_cancel(self): + def test_controller_scheduled_jobs_cancel(self): res = self.test_client.delete( "/scheduled-jobs/scheduled_echo", json={ @@ -118,7 +85,7 @@ def test_cancel(self): self.assertEqual(data, {"data": []}) - def test_create_without_command(self): + def test_controller_scheduled_jobs_create_without_command(self): res = self.test_client.post( "/scheduled-jobs/", json={ @@ -130,7 +97,7 @@ def test_create_without_command(self): self.assertEqual(res.status, "400 BAD REQUEST") self.assertEqual(data, {"error": "Request body contains no `command`"}) - def test_create_without_schedule(self): + def test_controller_scheduled_jobs_create_without_schedule(self): res = self.test_client.post( "/scheduled-jobs/", json={ @@ -142,7 +109,7 @@ def test_create_without_schedule(self): self.assertEqual(res.status, "400 BAD REQUEST") self.assertEqual(data, {"error": "Request body contains no `schedule`"}) - def test_create_with_unknown_group(self): + def test_controller_scheduled_jobs_create_with_unknown_group(self): res = self.test_client.post( "/scheduled-jobs/", json={ @@ -156,7 +123,7 @@ def test_create_with_unknown_group(self): self.assertEqual(res.status, "404 NOT FOUND") self.assertEqual(data, {"error": "Unknown group `unknown`"}) - def test_create_failed_by_scheduler_service(self): + def test_controller_scheduled_jobs_create_failed_by_scheduler_service(self): with patch.object(self.scheduler_service, "add", return_value=False): res = self.test_client.post( "/scheduled-jobs/", diff --git a/zapusk/server/controller_testcase.py b/zapusk/server/controller_testcase.py new file mode 100644 index 0000000..dd7687f --- /dev/null +++ b/zapusk/server/controller_testcase.py @@ -0,0 +1,58 @@ +from unittest import TestCase +from testfixtures import Replacer, TempDirectory + +from zapusk.services import ( + ConfigService, + SchedulerService, + ExecutorManagerService, + ExecutorManagerKawkaBackend, +) + +from .api import create_app + + +class ControllerTestCase(TestCase): + maxDiff = None + config_data = "" + + def before_create_services(self): ... + + def setUp(self) -> None: + self.replace = Replacer() + + self.temp_dir = TempDirectory() + self.config_file = self.temp_dir / "config.yml" + self.config_file.write_text(self.config_data) + + self.before_create_services() + + self.executor_manager_service = ExecutorManagerService( + backend=ExecutorManagerKawkaBackend(), + ) + self.config_service = ConfigService( + config_path=f"{self.temp_dir.path}/config.yml" + ) + self.scheduler_service = SchedulerService( + config_service=self.config_service, + executor_manager_service=self.executor_manager_service, + ) + self.scheduler_service.start() + + self.app = create_app( + executor_manager_service=self.executor_manager_service, + config_service=self.config_service, + scheduler_service=self.scheduler_service, + ) + self.test_client = self.app.test_client() + + def tearDown(self) -> None: + self.executor_manager_service.terminate() + self.scheduler_service.terminate() + self.temp_dir.cleanup() + self.replace.restore() + + def write_config(self, data): + self.config_file.write_text(data) + + def replace_in_environ(self, key, value): + self.replace.in_environ(key, value) diff --git a/zapusk/services/config/config_parser_test.py b/zapusk/services/config/config_parser_test.py index 15569e4..c2b2043 100644 --- a/zapusk/services/config/config_parser_test.py +++ b/zapusk/services/config/config_parser_test.py @@ -1,4 +1,5 @@ import pytest +from testfixtures import Replacer import yaml from zapusk.services.config.constants import DEFAULT_COLORS @@ -36,6 +37,7 @@ "name": "Sleep Timer", "id": "sleep", "command": "sleep 10", + "cwd": "/home/", "group": "default", "args_command": None, } @@ -70,6 +72,7 @@ "name": "Sleep Timer", "id": "sleep", "command": "sleep 10", + "cwd": "/home/", "group": "awesome_group", "args_command": None, } @@ -122,6 +125,7 @@ "name": "Sleep Timer", "id": "sleep", "command": "sleep 10", + "cwd": "/home/", "group": "default", "args_command": None, "on_fail": "echo job_fail", @@ -156,6 +160,7 @@ "name": "Sleep Timer", "id": "sleep", "command": "sleep 10", + "cwd": "/home/", "group": "default", "args_command": None, } @@ -173,11 +178,15 @@ ], ) def test_config_parser_should_parse_config(config_yaml, expected_result): + replace = Replacer() + replace.in_environ("HOME", "/home/") + config_parser = ConfigParser() config_data = yaml.safe_load(config_yaml) res = config_parser.parse(config_data) assert res == expected_result + replace.restore() #################################### diff --git a/zapusk/services/config/service.py b/zapusk/services/config/service.py index 214e2c1..1a518ec 100644 --- a/zapusk/services/config/service.py +++ b/zapusk/services/config/service.py @@ -12,7 +12,7 @@ class ConfigService: - config_path: str + config_path: str | None def __init__( self, @@ -29,7 +29,9 @@ def __get_config_path(self, config_path): Returns a path to the config file considering evnironment configuration """ if config_path: - return os.path.expanduser(config_path) + if isfile(config_path): + return os.path.expanduser(config_path) + return None config_dir = os.path.join( os.environ.get("APPDATA") @@ -50,10 +52,13 @@ def __get_config_path(self, config_path): logger.debug(f"Loaded config file: {config_dir}/config.yml") return f"{config_dir}/config.yml" else: - raise FileExistsError("Config not found") + return None def get_config(self): - config = self.file_reader.read(self.config_path) + if self.config_path: + config = self.file_reader.read(self.config_path) + else: + config = {} return self.parser.parse(config) def list_jobs(self): diff --git a/zapusk/services/config/service_test.py b/zapusk/services/config/service_test.py index 0222c92..69309dc 100644 --- a/zapusk/services/config/service_test.py +++ b/zapusk/services/config/service_test.py @@ -1,10 +1,19 @@ from unittest import TestCase -from testfixtures import TempDirectory, replace_in_environ +from testfixtures import Replacer, TempDirectory, replace_in_environ + +from zapusk.services.config.constants import DEFAULT_COLORS from .service import ConfigService class TestConfigService(TestCase): + def setUp(self) -> None: + self.r = Replacer() + self.r.in_environ("HOME", "/home/") + + def tearDown(self) -> None: + self.r.restore() + def test_config_service_should_return_jobs(self): config_service = ConfigService(config_path="./config.example.yaml") jobs = config_service.list_jobs() @@ -17,6 +26,7 @@ def test_config_service_should_return_jobs(self): "id": "sleep_10", "group": "default", "command": "sleep 10", + "cwd": "/var/", "args_command": None, }, ) @@ -28,6 +38,7 @@ def test_config_service_should_return_jobs(self): "id": "sleep_30", "group": "parallel", "command": "sleep 30", + "cwd": "/home/", "args_command": None, }, ) @@ -39,6 +50,7 @@ def test_config_service_should_return_jobs(self): "id": "sleep", "group": "sequential", "command": "sleep $1", + "cwd": "/home/", "args_command": "zenity --entry --text 'Sleep Time'", }, ) @@ -121,6 +133,7 @@ def test_config_service_should_return_job(self): "id": "sleep_10", "group": "default", "command": "sleep 10", + "cwd": "/var/", "args_command": None, }, ) @@ -176,3 +189,25 @@ def test_config_path_fail(self): ConfigService() except FileExistsError as ex: self.assertEqual(ex.args[0], "Config not found") + + def test_config_should_contain_only_defaults_if_config_file_does_not_exist(self): + config_service = ConfigService( + config_path="/home/leonid_brezhnev/plenum/config.yaml" + ) + config = config_service.get_config() + + self.assertEqual(len(config.job_groups), 1) + self.assertEqual( + config.job_groups["default"], + { + "id": "default", + "parallel": 10, + "on_finish": None, + "on_fail": None, + }, + ) + self.assertEqual(len(config.jobs), 0) + self.assertEqual(config.port, 9876) + self.assertEqual(config.colors, DEFAULT_COLORS) + + pass diff --git a/zapusk/services/executor_manager/backends/kawka/args_consumer.py b/zapusk/services/executor_manager/backends/kawka/args_consumer.py index 8d48f86..8690fe0 100644 --- a/zapusk/services/executor_manager/backends/kawka/args_consumer.py +++ b/zapusk/services/executor_manager/backends/kawka/args_consumer.py @@ -1,7 +1,9 @@ +import os import logging from datetime import datetime import subprocess + from zapusk.kawka import Consumer from zapusk.models import Job @@ -24,6 +26,8 @@ def process(self, job: Job): shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env={**os.environ}, + cwd=job.cwd, ) exit_code = proc.wait() out, err = proc.communicate() diff --git a/zapusk/services/executor_manager/backends/kawka/consumer_test.py b/zapusk/services/executor_manager/backends/kawka/consumer_test.py index 05f16e3..1820871 100644 --- a/zapusk/services/executor_manager/backends/kawka/consumer_test.py +++ b/zapusk/services/executor_manager/backends/kawka/consumer_test.py @@ -1,3 +1,4 @@ +import os from time import sleep from unittest import TestCase, mock from testfixtures.mock import call @@ -16,8 +17,12 @@ def setUp(self): self.Popen = MockPopen() self.r = Replacer() self.r.replace("subprocess.Popen", self.Popen) + self.r.in_environ("HOME", "/home/") self.addCleanup(self.r.restore) + def tearDown(self) -> None: + self.r.restore() + def test_should_get_args_and_run_job(self): input_producer = Producer(name="input_producer") @@ -45,6 +50,8 @@ def test_should_get_args_and_run_job(self): call.Popen( "get_args", shell=True, + env={**os.environ}, + cwd="/home/", stdout=-1, stderr=-1, ), @@ -54,6 +61,8 @@ def test_should_get_args_and_run_job(self): call.Popen( "my_command hello world", shell=True, + env={**os.environ}, + cwd="/home/", stdout=mock.ANY, stderr=mock.ANY, ), diff --git a/zapusk/services/executor_manager/backends/kawka/executor.py b/zapusk/services/executor_manager/backends/kawka/executor.py index 4efab99..b9b90b2 100644 --- a/zapusk/services/executor_manager/backends/kawka/executor.py +++ b/zapusk/services/executor_manager/backends/kawka/executor.py @@ -1,3 +1,4 @@ +import os import logging import subprocess from time import time @@ -33,6 +34,8 @@ def process(self, job: Job): shell=True, stdout=logfile, stderr=logfile, + env={**os.environ}, + cwd=job.cwd, ) job.pid = proc.pid @@ -53,6 +56,8 @@ def process(self, job: Job): subprocess.Popen( on_finish.format(job=job), shell=True, + env={**os.environ}, + cwd=job.cwd, ) else: @@ -64,6 +69,8 @@ def process(self, job: Job): subprocess.Popen( on_fail.format(job=job), shell=True, + env={**os.environ}, + cwd=job.cwd, ) logger.info(f"{self.name} failed {job} job") diff --git a/zapusk/services/executor_manager/backends/kawka/executor_test.py b/zapusk/services/executor_manager/backends/kawka/executor_test.py index 3224c51..6b641de 100644 --- a/zapusk/services/executor_manager/backends/kawka/executor_test.py +++ b/zapusk/services/executor_manager/backends/kawka/executor_test.py @@ -1,3 +1,4 @@ +import os from unittest import TestCase, mock from testfixtures.mock import call from testfixtures import Replacer @@ -15,6 +16,7 @@ def setUp(self): self.Popen = MockPopen() self.r = Replacer() self.r.replace("subprocess.Popen", self.Popen) + self.r.in_environ("HOME", "/home/") self.addCleanup(self.r.restore) def test_consumer_should_run_command(self): @@ -34,6 +36,8 @@ def test_consumer_should_run_command(self): call.Popen( "echo 1", shell=True, + env={**os.environ}, + cwd="/home/", stdout=mock.ANY, stderr=mock.ANY, ), @@ -60,6 +64,8 @@ def test_consumer_should_run_on_finish_callback(self): call.Popen( "echo 1", shell=True, + env={**os.environ}, + cwd="/home/", stdout=mock.ANY, stderr=mock.ANY, ), @@ -69,6 +75,8 @@ def test_consumer_should_run_on_finish_callback(self): self.Popen.all_calls[2], call.Popen( "echo finish", + env={**os.environ}, + cwd="/home/", shell=True, ), ) @@ -92,6 +100,8 @@ def test_consumer_should_run_on_finish_group_callback(self): self.Popen.all_calls[0], call.Popen( "echo 1", + env={**os.environ}, + cwd="/home/", shell=True, stdout=mock.ANY, stderr=mock.ANY, @@ -102,6 +112,8 @@ def test_consumer_should_run_on_finish_group_callback(self): self.Popen.all_calls[2], call.Popen( "echo finish", + env={**os.environ}, + cwd="/home/", shell=True, ), ) @@ -132,6 +144,8 @@ def test_consumer_should_run_on_finish_job_callback_if_both_job_and_group_are_de call.Popen( "echo 1", shell=True, + env={**os.environ}, + cwd="/home/", stdout=mock.ANY, stderr=mock.ANY, ), @@ -141,6 +155,8 @@ def test_consumer_should_run_on_finish_job_callback_if_both_job_and_group_are_de self.Popen.all_calls[2], call.Popen( "echo finish", + env={**os.environ}, + cwd="/home/", shell=True, ), ) @@ -164,6 +180,8 @@ def test_consumer_should_run_on_fail_callback(self): self.Popen.all_calls[0], call.Popen( "exit 1", + env={**os.environ}, + cwd="/home/", shell=True, stdout=mock.ANY, stderr=mock.ANY, @@ -174,6 +192,8 @@ def test_consumer_should_run_on_fail_callback(self): self.Popen.all_calls[2], call.Popen( "echo fail", + env={**os.environ}, + cwd="/home/", shell=True, ), ) @@ -197,6 +217,8 @@ def test_consumer_should_run_group_on_fail_callback(self): self.Popen.all_calls[0], call.Popen( "exit 1", + env={**os.environ}, + cwd="/home/", shell=True, stdout=mock.ANY, stderr=mock.ANY, @@ -207,6 +229,8 @@ def test_consumer_should_run_group_on_fail_callback(self): self.Popen.all_calls[2], call.Popen( "echo fail", + env={**os.environ}, + cwd="/home/", shell=True, ), ) @@ -234,6 +258,8 @@ def test_consumer_should_run_on_fail_job_callback_if_both_job_and_group_callback self.Popen.all_calls[0], call.Popen( "exit 1", + env={**os.environ}, + cwd="/home/", shell=True, stdout=mock.ANY, stderr=mock.ANY, @@ -244,6 +270,8 @@ def test_consumer_should_run_on_fail_job_callback_if_both_job_and_group_callback self.Popen.all_calls[2], call.Popen( "echo fail", + env={**os.environ}, + cwd="/home/", shell=True, ), ) @@ -264,6 +292,8 @@ def test_consumer_should_run_command_with_args(self): self.Popen.all_calls[0], call.Popen( "echo 1 2 3", + env={**os.environ}, + cwd="/home/", shell=True, stdout=mock.ANY, stderr=mock.ANY, @@ -287,6 +317,8 @@ def test_consumer_should_fail_command(self): self.Popen.all_calls[0], call.Popen( "exit 1", + env={**os.environ}, + cwd="/home/", shell=True, stdout=mock.ANY, stderr=mock.ANY, diff --git a/zapusk/services/scheduler_service/service_test.py b/zapusk/services/scheduler_service/service_test.py index f64632f..6f14412 100644 --- a/zapusk/services/scheduler_service/service_test.py +++ b/zapusk/services/scheduler_service/service_test.py @@ -24,6 +24,8 @@ def add(self): class TestSchedulerService(TestCase): + maxDiff = None + def setUp(self) -> None: self.temp_dir = TempDirectory() self.config_file = self.temp_dir / "config.yml" @@ -38,6 +40,7 @@ def setUp(self) -> None: self.r = Replacer() self.r.replace("zapusk.services.scheduler_service.service.datetime", self.d) self.r.replace("zapusk.models.scheduled_job.datetime", self.d) + self.r.in_environ("HOME", self.temp_dir.path) def tearDown(self) -> None: self.temp_dir.cleanup() @@ -111,6 +114,7 @@ def test_scheduler_service_should_list_all_scheduled_jobs(self): "id": "1", "name": "1", "command": "echo 1", + "cwd": self.temp_dir.path, "group": "default", "args_command": None, "on_finish": None, @@ -121,6 +125,7 @@ def test_scheduler_service_should_list_all_scheduled_jobs(self): "id": "2", "name": "2", "command": "echo 2", + "cwd": self.temp_dir.path, "group": "default", "args_command": None, "on_finish": None, @@ -163,6 +168,7 @@ def test_scheduler_service_should_delete_scheduled_jobs(self): "id": "2", "name": "2", "command": "echo 2", + "cwd": self.temp_dir.path, "group": "default", "args_command": None, "on_finish": None, @@ -207,6 +213,7 @@ def test_scheduler_service_delete_should_ignore_unknown_jobs(self): "id": "1", "name": "1", "command": "echo 1", + "cwd": self.temp_dir.path, "group": "default", "args_command": None, "on_finish": None, @@ -217,6 +224,7 @@ def test_scheduler_service_delete_should_ignore_unknown_jobs(self): "id": "2", "name": "2", "command": "echo 2", + "cwd": self.temp_dir.path, "group": "default", "args_command": None, "on_finish": None, From f291b81787dea106ddd11221b8668d25669309d0 Mon Sep 17 00:00:00 2001 From: Anton Shuvalov Date: Tue, 16 Jul 2024 01:29:31 +0700 Subject: [PATCH 2/2] Update readme --- README.md | 65 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index d00f98d..e17293a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,20 @@ # Zapusk -![Zapusk ScreenShot](.imgs/zapusk.png) +![Zapusk Screenshot](.imgs/zapusk.png) -Zapusk is a job runner for desktop environments. It helps you manage background tasks with features like pre-configured job execution, background shell commands, scheduling with cron-like syntax, log tailing, and notifications. It also provides detailed JSON output for easy data manipulation and analysis. +Zapusk is a versatile job runner designed for desktop environments. It simplifies the process of managing background tasks by providing robust features such as pre-configured job execution, background shell command execution, cron-like scheduling, log tailing, and notifications. Zapusk's detailed JSON output also enables powerful data manipulation and analysis when paired with tools like jq. +## Table of Contents + +- [Key Features](#key-features) +- [Installation](#installation) +- [Usage](#usage) + - [Basic Commands](#basic-commands) + - [Advanced Usage](#advanced-usage) +- [Configuration](#configuration) +- [Examples](#examples) +- [Contributing](#contributing) +- [License](#license) ## Key Features @@ -15,6 +26,7 @@ Zapusk is a job runner for desktop environments. It helps you manage background - **Job Groups:** Share settings like callbacks and parallelism between jobs. - **Colored JSON Output:** Easily readable JSON output. - **Waybar Integration:** Display job statuses and notifications on Waybar. +- **Custom Working Directory:** Run scripts and callbacks in a specified working directory. ## Installation @@ -26,8 +38,9 @@ pip install zapusk ## Usage -Zapusk requires `zapusk-server` to be started. Zapusk offers a command-line interface for managing and executing jobs. -Here's a quick reference: +Zapusk requires `zapusk-server` to be started. Zapusk offers a command-line interface for managing and executing jobs. Here's a quick reference: + +### Basic Commands ```sh Usage: @@ -51,34 +64,36 @@ Options: -n --name= Name for a command. -g --group= Job group to run the command in. -t --tail Tail logfile immediately. +``` -Examples: +### Examples - # Run npm install in the background - zapusk exec "npm i" +```sh +# Run npm install in the background +zapusk exec "npm i" - # Run pytest and tail its log - zapusk exec "pytest -v" -t +# Run pytest and tail its log +zapusk exec "pytest -v" -t - # Schedule a command to run every minute - zapusk exec "ping -c4 google.com" --schedule "*/1 * * * *" +# Schedule a command to run every minute +zapusk exec "ping -c4 google.com" --schedule "*/1 * * * *" - # Run a job defined in ~/.config/zapusk/config.yaml - zapusk run youtube_dl +# Run a job defined in ~/.config/zapusk/config.yaml +zapusk run youtube_dl - # Cancel a job by its ID - zapusk cancel 42 +# Cancel a job by its ID +zapusk cancel 42 - # See logs for a job by its ID - zapusk tail 42 +# See logs for a job by its ID +zapusk tail 42 ``` -## Example Configuration +## Configuration -Here is an example configuration file for Zapusk. It defines job groups and individual jobs, specifying commands, schedules, and notifications. +Here is an example configuration file for Zapusk. It defines job groups and individual jobs, specifying commands, schedules, notifications, and working directories. ```yaml -# Port server starts on and client call to +# The port the server starts on and the client connects to port: 9876 # Enable colored JSON output @@ -103,6 +118,7 @@ jobs: id: unsplash args_command: "zenity --entry --text 'Collection ID'" command: ~/.bin/jobs/unsplash_dl.sh + cwd: /path/to/working/directory - name: Sleep id: sleep @@ -132,6 +148,7 @@ jobs: id: unsplash args_command: "zenity --entry --text 'Collection ID'" command: ~/.bin/jobs/unsplash_wallpaper_collection_download.sh + cwd: /path/to/working/directory on_finish: notify-send -a "Zapusk" "Wallpapers downloaded" --icon kitty on_fail: notify-send -a "Zapusk" "Wallpaper download failed" --icon kitty ``` @@ -210,11 +227,12 @@ jobs: id: sleep group: sleep command: ~/.bin/jobs/sleep + cwd: /path/to/working/directory on_finish: notify-send -a "zapusk" "Job Finished" "{job.name} has finished" --icon kitty on_fail: notify-send -a "zapusk" "Job Failed" "{job.name} has failed" --icon kitty ``` -### Waybar Integration +## Waybar Integration Zapusk integrates with Waybar to display job statuses and notifications directly on your desktop. @@ -230,10 +248,7 @@ Zapusk integrates with Waybar to display job statuses and notifications directly } ``` -## Contribution - -We welcome contributions! If you find a bug or have an idea for improvement, please open an issue or submit a pull request on our GitHub repository. ## License -Zapusk is licensed under the MIT License. See the [LICENSE](LICENSE) file for more information. +Zapusk is licensed under the MIT License. See the [LICENSE](LICENSE.md) file for more information.