diff options
author | Oliver Poignant <oliver@poignant.se> | 2017-01-03 21:18:54 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-01-03 21:18:54 +0100 |
commit | d0c3f84a9be3f9ca4b2fbf63edf5951d3be1ea65 (patch) | |
tree | 93ea57bc3064fd10d37b9b8d3db07a91b57c499a | |
parent | 4d1d57104e2f3ab4dd4bd411a482e0f19565a433 (diff) | |
parent | f48093c4415c04b070886c19bdc2fcc7c12fcbd0 (diff) | |
download | Git-Auto-Deploy-d0c3f84a9be3f9ca4b2fbf63edf5951d3be1ea65.zip Git-Auto-Deploy-d0c3f84a9be3f9ca4b2fbf63edf5951d3be1ea65.tar.gz Git-Auto-Deploy-d0c3f84a9be3f9ca4b2fbf63edf5951d3be1ea65.tar.bz2 |
Merge pull request #159 from olipo186/python3
Refactoring and python 3 support
-rw-r--r-- | README.md | 4 | ||||
-rw-r--r-- | docs/Configuration.md | 1 | ||||
-rw-r--r-- | docs/Start automatically on boot.md | 4 | ||||
-rw-r--r-- | gitautodeploy/__init__.py | 7 | ||||
-rw-r--r-- | gitautodeploy/__main__.py | 2 | ||||
-rw-r--r-- | gitautodeploy/cli/__init__.py | 2 | ||||
-rw-r--r-- | gitautodeploy/cli/config.py | 9 | ||||
-rw-r--r-- | gitautodeploy/events.py | 6 | ||||
-rw-r--r-- | gitautodeploy/gitautodeploy.py | 318 | ||||
-rw-r--r-- | gitautodeploy/httpserver.py | 69 | ||||
-rw-r--r-- | gitautodeploy/parsers/__init__.py | 12 | ||||
-rw-r--r-- | gitautodeploy/parsers/bitbucket.py | 2 | ||||
-rw-r--r-- | gitautodeploy/parsers/coding.py | 2 | ||||
-rw-r--r-- | gitautodeploy/parsers/generic.py | 2 | ||||
-rw-r--r-- | gitautodeploy/parsers/github.py | 2 | ||||
-rw-r--r-- | gitautodeploy/parsers/gitlab.py | 2 | ||||
-rw-r--r-- | gitautodeploy/parsers/gitlabci.py | 2 | ||||
-rw-r--r-- | gitautodeploy/wrappers/__init__.py | 4 | ||||
-rw-r--r-- | gitautodeploy/wrappers/git.py | 8 | ||||
-rw-r--r-- | gitautodeploy/wrappers/process.py | 4 | ||||
-rw-r--r-- | gitautodeploy/wsserver.py | 21 | ||||
-rwxr-xr-x | platforms/linux/initfiles/debianLSBInitScripts/git-auto-deploy | 2 | ||||
-rw-r--r-- | platforms/linux/initfiles/systemd/git-auto-deploy.service | 2 |
23 files changed, 242 insertions, 245 deletions
@@ -67,11 +67,11 @@ Copy of the sample config and modify it. [Read more about the configuration opti Start ```Git-Auto-Deploy``` manually using; - python gitautodeploy --config config.json + python -m gitautodeploy --config config.json To start ```Git-Auto-Deploy``` automatically on boot, open crontab in edit mode using ```crontab -e``` and add the entry below. - @reboot /usr/bin/python /path/to/Git-Auto-Deploy/gitautodeploy --daemon-mode --quiet --config /path/to/git-auto-deploy.conf.json + @reboot /usr/bin/python -m /path/to/Git-Auto-Deploy/gitautodeploy --daemon-mode --quiet --config /path/to/git-auto-deploy.conf.json You can also configure ```Git-Auto-Deploy``` to start on boot using a init.d-script (for Debian and Sys-V like init systems) or a service for systemd. [Read more about starting Git-Auto-Deploy automatically using init.d or systemd](./docs/Start automatically on boot.md). diff --git a/docs/Configuration.md b/docs/Configuration.md index f2914f6..2e92c76 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -15,7 +15,6 @@ Command line option | Environment variable | Config attribute | Description --log-file <path> | GAD_LOG_FILE | logfilepath | Specify a log file --host <host> | GAD_HOST | host | Address to bind to --port <port> | GAD_PORT | port | Port to bind to ---force | GAD_FORCE | | Kill any process using the configured port --ssh-keyscan | GAD_SSH_KEYSCAN | | Scan repository hosts for ssh keys and add them to $HOME/.ssh/known_hosts # Configuration file options diff --git a/docs/Start automatically on boot.md b/docs/Start automatically on boot.md index f0f3ffb..30d5702 100644 --- a/docs/Start automatically on boot.md +++ b/docs/Start automatically on boot.md @@ -2,14 +2,14 @@ ```Git-Auto-Deploy``` can be automatically started at boot time using various techniques. Below you'll find a couple of suggested approaches with instructions. -The following instructions assumes that you are running ```Git-Auto-Deploy``` from a clone of this repository. In such a case, ```Git-Auto-Deploy``` is started by invoking ```python``` and referencing the ```gitautodeploy``` python module which is found in the cloned repository. Such a command can look like ```python /path/to/Git-Auto-Deploy/gitautodeploy --daemon-mode```. +The following instructions assumes that you are running ```Git-Auto-Deploy``` from a clone of this repository. In such a case, ```Git-Auto-Deploy``` is started by invoking ```python -m``` and referencing the ```gitautodeploy``` python module which is found in the cloned repository. Such a command can look like ```python -m /path/to/Git-Auto-Deploy/gitautodeploy --daemon-mode```. If you have used any of the alternative installation methods (install with pip or as a debian package), you will instead start ```Git-Auto-Deploy``` using a installed executable. ```Git-Auto-Deploy``` would then be started using a command like ```git-auto-deploy --daemon-mode``` instead. If you have installed ```Git-Auto-Deploy``` in this way, you will need to modify the paths and commands used in the instructions below. ## Crontab The easiest way to configure your system to automatically start ```Git-Auto-Deploy``` after a reboot is using crontab. Open crontab in edit mode using ```crontab -e``` and add the following: - @reboot /usr/bin/python /path/to/Git-Auto-Deploy/gitautodeploy --daemon-mode --quiet + @reboot /usr/bin/python -m /path/to/Git-Auto-Deploy/gitautodeploy --daemon-mode --quiet ## Debian and Sys-V like init system. diff --git a/gitautodeploy/__init__.py b/gitautodeploy/__init__.py index c8a461c..8f762c3 100644 --- a/gitautodeploy/__init__.py +++ b/gitautodeploy/__init__.py @@ -1,5 +1,2 @@ -from wrappers import * -from lock import * -from parsers import * -from gitautodeploy import * -from cli import *
\ No newline at end of file +from .gitautodeploy import * +from .cli import *
\ No newline at end of file diff --git a/gitautodeploy/__main__.py b/gitautodeploy/__main__.py index f97e44f..0070f6a 100644 --- a/gitautodeploy/__main__.py +++ b/gitautodeploy/__main__.py @@ -2,4 +2,6 @@ from gitautodeploy import main if __name__ == '__main__': + import sys, os + sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) main() diff --git a/gitautodeploy/cli/__init__.py b/gitautodeploy/cli/__init__.py index 1725ebe..27c9ec6 100644 --- a/gitautodeploy/cli/__init__.py +++ b/gitautodeploy/cli/__init__.py @@ -1 +1 @@ -from config import * +from .config import * diff --git a/gitautodeploy/cli/config.py b/gitautodeploy/cli/config.py index e302da4..f87057e 100644 --- a/gitautodeploy/cli/config.py +++ b/gitautodeploy/cli/config.py @@ -6,7 +6,6 @@ def get_config_defaults(): config['daemon-mode'] = False config['config'] = None config['ssh-keyscan'] = False - config['force'] = False config['ssl'] = False config['ssl-pem-file'] = '~/.gitautodeploy.pem' config['pidfilepath'] = '~/.gitautodeploy.pid' @@ -52,9 +51,6 @@ def get_config_from_environment(): if 'GAD_SSH_KEYSCAN' in os.environ: config['ssh-keyscan'] = True - if 'GAD_FORCE' in os.environ: - config['force'] = True - if 'GAD_SSL' in os.environ: config['ssl'] = True @@ -100,11 +96,6 @@ def get_config_from_argv(argv): dest="ssh-keyscan", action="store_true") - parser.add_argument("--force", - help="kill any process using the configured port", - dest="force", - action="store_true") - parser.add_argument("--pid-file", help="specify a custom pid file", dest="pidfilepath", diff --git a/gitautodeploy/events.py b/gitautodeploy/events.py index bbf93a2..cf777a6 100644 --- a/gitautodeploy/events.py +++ b/gitautodeploy/events.py @@ -67,6 +67,7 @@ class SystemEvent(object): self.register_message(message, "ERROR") def log_critical(self, message): + self.logger.critical(message) self.register_message(message, "CRITICAL") def update(self): @@ -147,4 +148,7 @@ class EventStore(object): self.update_observers(action, message) def dict_repr(self): - return map(lambda x: x.dict_repr(), self.actions) + action_repr = [] + for action in self.actions: + action_repr.append(action.dict_repr()) + return action_repr diff --git a/gitautodeploy/gitautodeploy.py b/gitautodeploy/gitautodeploy.py index c0d5b21..c7ebf3a 100644 --- a/gitautodeploy/gitautodeploy.py +++ b/gitautodeploy/gitautodeploy.py @@ -11,16 +11,20 @@ class LogInterface(object): for line in msg.strip().split("\n"): self.level(line) + def flush(self): + pass class GitAutoDeploy(object): _instance = None - _server = None + _http_server = None + _ws_server = None _config = {} _port = None _pid = None _event_store = None _default_stdout = None _default_stderr = None + _startup_event = None def __new__(cls, *args, **kwargs): """Overload constructor to enable singleton access""" @@ -29,61 +33,16 @@ class GitAutoDeploy(object): cls, *args, **kwargs) return cls._instance - @staticmethod - def debug_diagnosis(port): - """Display information about what process is using the specified port.""" - import logging - logger = logging.getLogger() - - pid = GitAutoDeploy.get_pid_on_port(port) - if pid is False: - logger.warning('Unable to determine what PID is using port %s' % port) - return - - logger.info('Process with PID %s is using port %s' % (pid, port)) - with open("/proc/%s/cmdline" % pid) as f: - cmdline = f.readlines() - logger.info('Process with PID %s was started using the command: %s' % (pid, cmdline[0].replace('\x00', ' '))) - - @staticmethod - def get_pid_on_port(port): - """Determine what process (PID) is using a specific port.""" - import os - - with open("/proc/net/tcp", 'r') as f: - file_content = f.readlines()[1:] - - pids = [int(x) for x in os.listdir('/proc') if x.isdigit()] - conf_port = str(port) - mpid = False - - for line in file_content: - if mpid is not False: - break - - _, laddr, _, _, _, _, _, _, _, inode = line.split()[:10] - decport = str(int(laddr.split(':')[1], 16)) - - if decport != conf_port: - continue - - for pid in pids: - try: - path = "/proc/%s/fd" % pid - if os.access(path, os.R_OK) is False: - continue + def __init__(self): + from .events import EventStore, StartupEvent - for fd in os.listdir(path): - cinode = os.readlink("/proc/%s/fd/%s" % (pid, fd)) - minode = cinode.split(":") - - if len(minode) == 2 and minode[1][1:-1] == inode: - mpid = pid - - except Exception as e: - pass + # Setup an event store instance that can keep a global record of events + self._event_store = EventStore() + self._event_store.register_observer(self) - return mpid + # Create a startup event that can hold status and any error messages from the startup process + self._startup_event = StartupEvent() + self._event_store.register_action(self._startup_event) def clone_all_repos(self): """Iterates over all configured repositories and clones them to their @@ -91,7 +50,7 @@ class GitAutoDeploy(object): import os import re import logging - from wrappers import GitWrapper + from .wrappers import GitWrapper logger = logging.getLogger() if not 'repositories' in self._config: @@ -103,7 +62,6 @@ class GitAutoDeploy(object): # Only clone repositories with a configured path if 'url' not in repo_config: logger.critical("Repository has no configured URL") - self.close() self.exit() return @@ -120,7 +78,7 @@ class GitAutoDeploy(object): def ssh_key_scan(self): import re import logging - from wrappers import ProcessWrapper + from .wrappers import ProcessWrapper logger = logging.getLogger() for repository in self._config['repositories']: @@ -141,28 +99,6 @@ class GitAutoDeploy(object): else: logger.error('Could not find regexp match in path: %s' % repository['url']) - def kill_conflicting_processes(self): - """Attempt to kill any process already using the configured port.""" - import os - import logging - import signal - logger = logging.getLogger() - - pid = GitAutoDeploy.get_pid_on_port(self._config['port']) - - if pid is False: - logger.warning('No process is currently using port %s.' % self._config['port']) - return False - - if hasattr(signal, 'SIGKILL'): - os.kill(pid, signal.SIGKILL) - elif hasattr(signal, 'SIGHUP'): - os.kill(pid, signal.SIGHUP) - else: - os.kill(pid, 1) - - return True - def create_pid_file(self): import os @@ -179,25 +115,10 @@ class GitAutoDeploy(object): if 'pidfilepath' in self._config and self._config['pidfilepath']: try: os.remove(self._config['pidfilepath']) - except OSError, e: + except OSError as e: if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory raise - def close(self): - import sys - import logging - logger = logging.getLogger() - logger.info('Goodbye') - self.remove_pid_file() - if 'intercept-stdout' in self._config and self._config['intercept-stdout']: - sys.stdout = self._default_stdout - sys.stderr = self._default_stderr - - def exit(self): - import sys - self.close() - sys.exit(0) - @staticmethod def create_daemon(): import os @@ -205,7 +126,7 @@ class GitAutoDeploy(object): try: # Spawn first child. Returns 0 in the child and pid in the parent. pid = os.fork() - except OSError, e: + except OSError as e: raise Exception("%s [%d]" % (e.strerror, e.errno)) # First child @@ -216,7 +137,7 @@ class GitAutoDeploy(object): # Spawn second child pid = os.fork() - except OSError, e: + except OSError as e: raise Exception("%s [%d]" % (e.strerror, e.errno)) if pid == 0: @@ -236,13 +157,16 @@ class GitAutoDeploy(object): def setup(self, config): """Setup an instance of GAD based on the provided config object.""" import sys - from BaseHTTPServer import HTTPServer import socket import os import logging - from lock import Lock - from httpserver import WebhookRequestHandlerFactory - from events import EventStore, StartupEvent + from .lock import Lock + from .httpserver import WebhookRequestHandlerFactory + + try: + from BaseHTTPServer import HTTPServer + except ImportError as e: + from http.server import HTTPServer # This solves https://github.com/olipo186/Git-Auto-Deploy/issues/118 try: @@ -257,12 +181,6 @@ class GitAutoDeploy(object): # Attatch config values to this instance self._config = config - self._event_store = EventStore() - self._event_store.register_observer(self) - - startup_event = StartupEvent() - self._event_store.register_action(startup_event) - # Set up logging logger = logging.getLogger() logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") @@ -296,13 +214,9 @@ class GitAutoDeploy(object): logger.addHandler(fileHandler) if 'ssh-keyscan' in self._config and self._config['ssh-keyscan']: - startup_event.log_info('Scanning repository hosts for ssh keys...') + self._startup_event.log_info('Scanning repository hosts for ssh keys...') self.ssh_key_scan() - if 'force' in self._config and self._config['force']: - startup_event.log_info('Attempting to kill any other process currently occupying port %s' % self._config['port']) - self.kill_conflicting_processes() - # Clone all repos once initially self.clone_all_repos() @@ -315,10 +229,10 @@ class GitAutoDeploy(object): sys.stderr = LogInterface(logger.error) if 'daemon-mode' in self._config and self._config['daemon-mode']: - startup_event.log_info('Starting Git Auto Deploy in daemon mode') + self._startup_event.log_info('Starting Git Auto Deploy in daemon mode') GitAutoDeploy.create_daemon() else: - startup_event.log_info('Git Auto Deploy started') + self._startup_event.log_info('Git Auto Deploy started') self._pid = os.getpid() self.create_pid_file() @@ -336,118 +250,186 @@ class GitAutoDeploy(object): # Create web hook request handler class WebhookRequestHandler = WebhookRequestHandlerFactory(self._config, self._event_store) - self._server = HTTPServer((self._config['host'], + # Create HTTP server + self._http_server = HTTPServer((self._config['host'], self._config['port']), WebhookRequestHandler) + # Start a web socket server if the web UI is enabled + #if self._config['web-ui']['enabled']: + + # try: + # from SimpleWebSocketServer import SimpleWebSocketServer + # from wsserver import WebSocketClientHandler + + # # Create web socket server + # self._ws_server = SimpleWebSocketServer(self._config['ws-host'], self._config['ws-port'], WebSocketClientHandler) + + # sa = self._ws_server.socket.getsockname() + # self._startup_event.log_info("Listening for web socket on %s port %s" % (sa[0], sa[1])) + + # except ImportError as e: + # self._startup_event.log_error("Unable to start web socket server due to a too old version of Python. Version => 2.7.9 is required.") + + # Setup SSL for HTTP server if 'ssl' in self._config and self._config['ssl']: import ssl logger.info("enabling ssl") - self._server.socket = ssl.wrap_socket(self._server.socket, + self._http_server.socket = ssl.wrap_socket(self._http_server.socket, certfile=os.path.expanduser(self._config['ssl-pem']), server_side=True) - sa = self._server.socket.getsockname() - startup_event.log_info("Listening on %s port %s" % (sa[0], sa[1])) - startup_event.address = sa[0] - startup_event.port = sa[1] - startup_event.notify() + sa = self._http_server.socket.getsockname() + self._startup_event.log_info("Listening for http connections on %s port %s" % (sa[0], sa[1])) + self._startup_event.address = sa[0] + self._startup_event.port = sa[1] + self._startup_event.notify() # Actual port bound to (nessecary when OS picks randomly free port) self._port = sa[1] - except socket.error, e: - startup_event.log_critical("Error on socket: %s" % e) - GitAutoDeploy.debug_diagnosis(self._config['port']) - + except socket.error as e: + self._startup_event.log_critical("Error on socket: %s" % e) sys.exit(1) - def serve_forever(self): - """Start listening for incoming requests.""" + def serve_http(self): import sys import socket import logging import os - from events import SystemEvent - - # Add script dir to sys path, allowing us to import sub modules even after changing cwd - sys.path.insert(1, os.path.dirname(os.path.realpath(__file__))) - - # Set CWD to public www folder. This makes the http server serve files from the wwwroot directory. - wwwroot = os.path.join(os.path.dirname(os.path.realpath(__file__)), "wwwroot") - os.chdir(wwwroot) - - # Set up logging - logger = logging.getLogger() + from .events import SystemEvent try: - self._server.serve_forever() + self._http_server.serve_forever() - except socket.error, e: - logger.critical("Error on socket: %s" % e) + except socket.error as e: event = SystemEvent() self._event_store.register_action(event) event.log_critical("Error on socket: %s" % e) sys.exit(1) - except KeyboardInterrupt, e: - logger.info('Requested close by keyboard interrupt signal') + except KeyboardInterrupt as e: event = SystemEvent() self._event_store.register_action(event) event.log_info('Requested close by keyboard interrupt signal') self.stop() self.exit() - def handle_request(self): + pass + + def serve_ws(self): + if not self._ws_server: + return + self._ws_server.serveforever() + + def serve_forever(self): """Start listening for incoming requests.""" import sys import socket import logging + import os + from .events import SystemEvent + import threading - # Set up logging - logger = logging.getLogger() + # Add script dir to sys path, allowing us to import sub modules even after changing cwd + sys.path.insert(1, os.path.dirname(os.path.realpath(__file__))) + + # Set CWD to public www folder. This makes the http server serve files from the wwwroot directory. + wwwroot = os.path.join(os.path.dirname(os.path.realpath(__file__)), "wwwroot") + os.chdir(wwwroot) + + t1 = threading.Thread(target=self.serve_http) + #t1.daemon = True + t1.start() + + + #t2 = threading.Thread(target=self.serve_ws) + #t1.daemon = True + #t2.start() + + # Wait for thread to finish without blocking main thread + while t1.isAlive: + t1.join(5) + + # Wait for thread to finish without blocking main thread + #while t2.isAlive: + # t2.join(5) + + + def handle_request(self): + """Start listening for incoming requests.""" + import sys + import socket + from .events import SystemEvent try: - self._server.handle_request() + self._http_server.handle_request() - except socket.error, e: - logger.critical("Error on socket: %s" % e) + except socket.error as e: + event = SystemEvent() + self._event_store.register_action(event) + event.log_critical("Error on socket: %s" % e) sys.exit(1) - except KeyboardInterrupt, e: - logger.info('Requested close by keyboard interrupt signal') + except KeyboardInterrupt as e: + event = SystemEvent() + self._event_store.register_action(event) + event.log_info('Requested close by keyboard interrupt signal') self.stop() self.exit() - def stop(self): - if self._server is None: - return - self._server.socket.close() - def signal_handler(self, signum, frame): - from events import SystemEvent - import logging - logger = logging.getLogger() + from .events import SystemEvent self.stop() + event = SystemEvent() + self._event_store.register_action(event) + + # Reload configuration on SIGHUP events (conventional for daemon processes) if signum == 1: self.setup(self._config) self.serve_forever() return + # Keyboard interrupt signal elif signum == 2: - logger.info('Requested close by keyboard interrupt signal') - event = SystemEvent() - self._event_store.register_action(event) - event.log_info('Requested close by keyboard interrupt signal') + event.log_info('Recieved keyboard interrupt signal (%s) from the OS, shutting down.' % signum) - elif signum == 6: - logger.info('Requested close by SIGABRT (process abort signal). Code 6.') - event = SystemEvent() - self._event_store.register_action(event) - event.log_info('Requested close by SIGABRT (process abort signal). Code 6.') + else: + event.log_info('Recieved signal (%s) from the OS, shutting down.' % signum) self.exit() + def stop(self): + """Stop all running TCP servers (HTTP and web socket servers)""" + + # Stop HTTP server if running + if self._http_server is not None: + + # Shut down the underlying TCP server + self._http_server.shutdown() + # Close the socket + self._http_server.socket.close() + + # Stop web socket server if running + if self._ws_server is not None: + self._ws_server.close() + + def exit(self): + import sys + import logging + logger = logging.getLogger() + logger.info('Goodbye') + + # Delete PID file + self.remove_pid_file() + + # Restore stdin and stdout + if 'intercept-stdout' in self._config and self._config['intercept-stdout']: + sys.stdout = self._default_stdout + sys.stderr = self._default_stderr + + sys.exit(0) + def main(): import signal from gitautodeploy import GitAutoDeploy diff --git a/gitautodeploy/httpserver.py b/gitautodeploy/httpserver.py index 38f28ff..5a06498 100644 --- a/gitautodeploy/httpserver.py +++ b/gitautodeploy/httpserver.py @@ -1,5 +1,4 @@ -from BaseHTTPServer import BaseHTTPRequestHandler - +from .parsers import CodingRequestParser, GitLabCIRequestParser, GitLabRequestParser, GitHubRequestParser, BitBucketRequestParser, GenericRequestParser class WebbhookRequestProcessor(object): @@ -7,10 +6,7 @@ class WebbhookRequestProcessor(object): """Parses the incoming request and attempts to determine whether it originates from GitHub, GitLab or any other known service.""" import json - import logging - import parsers - logger = logging.getLogger() payload = json.loads(request_body) if not isinstance(payload, dict): @@ -21,33 +17,33 @@ class WebbhookRequestProcessor(object): # Assume Coding if the X-Coding-Event HTTP header is set if 'x-coding-event' in request_headers: - return parsers.CodingRequestParser + return CodingRequestParser # Assume GitLab if the X-Gitlab-Event HTTP header is set elif 'x-gitlab-event' in request_headers: # Special Case for Gitlab CI if content_type == "application/json" and "build_status" in payload: - return parsers.GitLabCIRequestParser + return GitLabCIRequestParser else: - return parsers.GitLabRequestParser + return GitLabRequestParser # Assume GitHub if the X-GitHub-Event HTTP header is set elif 'x-github-event' in request_headers: - return parsers.GitHubRequestParser + return GitHubRequestParser # Assume BitBucket if the User-Agent HTTP header is set to # 'Bitbucket-Webhooks/2.0' (or something similar) elif user_agent and user_agent.lower().find('bitbucket') != -1: - return parsers.BitBucketRequestParser + return BitBucketRequestParser # This handles old GitLab requests and Gogs requests for example. elif content_type == "application/json": action.log_info("Received event from unknown origin.") - return parsers.GenericRequestParser + return GenericRequestParser action.log_error("Unable to recognize request origin. Don't know how to handle the request.") return @@ -58,8 +54,8 @@ class WebbhookRequestProcessor(object): import os import time import logging - from wrappers import GitWrapper - from lock import Lock + from .wrappers import GitWrapper + from .lock import Lock import json logger = logging.getLogger() @@ -146,6 +142,10 @@ class WebbhookRequestProcessor(object): result.append(repo_result) + action.log_info("Deploy commands were executed") + action.set_success(True) + action.update() + return result @@ -160,7 +160,7 @@ class WebhookRequestFilter(object): for filter in payload_filters: # All options specified in the filter must match - for filter_key, filter_value in filter.iteritems(): + for filter_key, filter_value in filter.items(): # Ignore filters with value None (let them pass) if filter_value == None: @@ -224,8 +224,8 @@ class WebhookRequestFilter(object): import os import time import logging - from wrappers import GitWrapper - from lock import Lock + from .wrappers import GitWrapper + from .lock import Lock import json logger = logging.getLogger() @@ -255,7 +255,10 @@ class WebhookRequestFilter(object): def WebhookRequestHandlerFactory(config, event_store): """Factory method for webhook request handler class""" - from SimpleHTTPServer import SimpleHTTPRequestHandler + try: + from SimpleHTTPServer import SimpleHTTPRequestHandler + except ImportError as e: + from http.server import SimpleHTTPRequestHandler class WebhookRequestHandler(SimpleHTTPRequestHandler, object): """Extends the BaseHTTPRequestHandler class and handles the incoming @@ -291,8 +294,7 @@ def WebhookRequestHandlerFactory(config, event_store): self.send_response(200, 'OK') self.send_header('Content-type', 'application/json') self.end_headers() - self.wfile.write(json.dumps(data)) - self.wfile.close() + self.wfile.write(json.dumps(data).encode('utf-8')) return return SimpleHTTPRequestHandler.do_GET(self) @@ -302,16 +304,17 @@ def WebhookRequestHandlerFactory(config, event_store): from threading import Timer import logging import json - from events import WebhookAction + from .events import WebhookAction + import threading logger = logging.getLogger() - content_length = int(self.headers.getheader('content-length')) - request_body = self.rfile.read(content_length) + content_length = int(self.headers.get('content-length')) + request_body = self.rfile.read(content_length).decode('utf-8') # Extract request headers and make all keys to lowercase (makes them easier to compare) request_headers = dict(self.headers) - request_headers = dict((k.lower(), v) for k, v in request_headers.iteritems()) + request_headers = dict((k.lower(), v) for k, v in request_headers.items()) action = WebhookAction(self.client_address, request_headers, request_body) action.set_waiting(True) @@ -365,15 +368,11 @@ def WebhookRequestHandlerFactory(config, event_store): action.log_warning("Request not valid") return - # Send HTTP response before the git pull and/or deploy commands? - #if not 'detailed-response' in self._config or not self._config['detailed-response']: + test_case['expected']['status'] = 200 + self.send_response(200, 'OK') self.send_header('Content-type', 'text/plain') self.end_headers() - self.finish() - self.connection.close() - - test_case['expected']['status'] = 200 if len(repo_configs) == 0: action.log_info("Filter does not match") @@ -382,7 +381,9 @@ def WebhookRequestHandlerFactory(config, event_store): action.log_info("Executing deploy commands") # Schedule the execution of the webhook (git pull and trigger deploy etc) - request_processor.execute_webhook(repo_configs, request_headers, request_body, action) + #request_processor.execute_webhook(repo_configs, request_headers, request_body, action) + thread = threading.Thread(target=request_processor.execute_webhook, args=[repo_configs, request_headers, request_body, action]) + thread.start() # Add additional test case data test_case['config'] = { @@ -392,11 +393,7 @@ def WebhookRequestHandlerFactory(config, event_store): 'deploy': 'echo test!' } - action.log_info("Deploy commands were executed") - action.set_success(True) - action.update() - - except ValueError, e: + except ValueError as e: self.send_error(400, 'Unprocessable request') action.log_warning('Unable to process incoming request from %s:%s' % (self.client_address[0], self.client_address[1])) test_case['expected']['status'] = 400 @@ -404,7 +401,7 @@ def WebhookRequestHandlerFactory(config, event_store): action.update() return - except Exception, e: + except Exception as e: if 'detailed-response' in self._config and self._config['detailed-response']: self.send_error(500, 'Unable to process request') diff --git a/gitautodeploy/parsers/__init__.py b/gitautodeploy/parsers/__init__.py index c993454..cecd1c7 100644 --- a/gitautodeploy/parsers/__init__.py +++ b/gitautodeploy/parsers/__init__.py @@ -1,6 +1,6 @@ -from bitbucket import BitBucketRequestParser -from github import GitHubRequestParser -from gitlab import GitLabRequestParser -from gitlabci import GitLabCIRequestParser -from generic import GenericRequestParser -from coding import CodingRequestParser
\ No newline at end of file +from .bitbucket import BitBucketRequestParser +from .github import GitHubRequestParser +from .gitlab import GitLabRequestParser +from .gitlabci import GitLabCIRequestParser +from .generic import GenericRequestParser +from .coding import CodingRequestParser
\ No newline at end of file diff --git a/gitautodeploy/parsers/bitbucket.py b/gitautodeploy/parsers/bitbucket.py index 80899fd..b80f6aa 100644 --- a/gitautodeploy/parsers/bitbucket.py +++ b/gitautodeploy/parsers/bitbucket.py @@ -1,4 +1,4 @@ -from common import WebhookRequestParser +from .common import WebhookRequestParser class BitBucketRequestParser(WebhookRequestParser): diff --git a/gitautodeploy/parsers/coding.py b/gitautodeploy/parsers/coding.py index 889c3ee..3d114d4 100644 --- a/gitautodeploy/parsers/coding.py +++ b/gitautodeploy/parsers/coding.py @@ -1,4 +1,4 @@ -from common import WebhookRequestParser +from .common import WebhookRequestParser class CodingRequestParser(WebhookRequestParser): diff --git a/gitautodeploy/parsers/generic.py b/gitautodeploy/parsers/generic.py index a93e90b..c71bced 100644 --- a/gitautodeploy/parsers/generic.py +++ b/gitautodeploy/parsers/generic.py @@ -1,4 +1,4 @@ -from common import WebhookRequestParser +from .common import WebhookRequestParser class GenericRequestParser(WebhookRequestParser): diff --git a/gitautodeploy/parsers/github.py b/gitautodeploy/parsers/github.py index bede7bb..5f2ba5d 100644 --- a/gitautodeploy/parsers/github.py +++ b/gitautodeploy/parsers/github.py @@ -1,4 +1,4 @@ -from common import WebhookRequestParser +from .common import WebhookRequestParser class GitHubRequestParser(WebhookRequestParser): diff --git a/gitautodeploy/parsers/gitlab.py b/gitautodeploy/parsers/gitlab.py index 5d0d348..6767907 100644 --- a/gitautodeploy/parsers/gitlab.py +++ b/gitautodeploy/parsers/gitlab.py @@ -1,4 +1,4 @@ -from common import WebhookRequestParser +from .common import WebhookRequestParser class GitLabRequestParser(WebhookRequestParser): diff --git a/gitautodeploy/parsers/gitlabci.py b/gitautodeploy/parsers/gitlabci.py index 22af596..eebf2bd 100644 --- a/gitautodeploy/parsers/gitlabci.py +++ b/gitautodeploy/parsers/gitlabci.py @@ -1,4 +1,4 @@ -from common import WebhookRequestParser +from .common import WebhookRequestParser class GitLabCIRequestParser(WebhookRequestParser): diff --git a/gitautodeploy/wrappers/__init__.py b/gitautodeploy/wrappers/__init__.py index d7df44b..aad49c9 100644 --- a/gitautodeploy/wrappers/__init__.py +++ b/gitautodeploy/wrappers/__init__.py @@ -1,2 +1,2 @@ -from git import * -from process import *
\ No newline at end of file +from .git import * +from .process import *
\ No newline at end of file diff --git a/gitautodeploy/wrappers/git.py b/gitautodeploy/wrappers/git.py index 910e9f6..1f7fe23 100644 --- a/gitautodeploy/wrappers/git.py +++ b/gitautodeploy/wrappers/git.py @@ -9,7 +9,7 @@ class GitWrapper(): def init(repo_config): """Init remote url of the repo from the git server""" import logging - from process import ProcessWrapper + from .process import ProcessWrapper import os import platform @@ -49,7 +49,7 @@ class GitWrapper(): def pull(repo_config): """Pulls the latest version of the repo from the git server""" import logging - from process import ProcessWrapper + from .process import ProcessWrapper import os import platform @@ -93,7 +93,7 @@ class GitWrapper(): def clone(repo_config): """Clones the latest version of the repo from the git server""" import logging - from process import ProcessWrapper + from .process import ProcessWrapper import os import platform @@ -127,7 +127,7 @@ class GitWrapper(): @staticmethod def deploy(repo_config): """Executes any supplied post-pull deploy command""" - from process import ProcessWrapper + from .process import ProcessWrapper import logging logger = logging.getLogger() diff --git a/gitautodeploy/wrappers/process.py b/gitautodeploy/wrappers/process.py index 30adc36..867e6cf 100644 --- a/gitautodeploy/wrappers/process.py +++ b/gitautodeploy/wrappers/process.py @@ -20,6 +20,10 @@ class ProcessWrapper(): p = Popen(*popenargs, **kwargs) stdout, stderr = p.communicate() + # Decode bytes to string (assume utf-8 encoding) + stdout = stdout.decode("utf-8") + stderr = stderr.decode("utf-8") + if stdout: for line in stdout.strip().split("\n"): logger.info(line) diff --git a/gitautodeploy/wsserver.py b/gitautodeploy/wsserver.py new file mode 100644 index 0000000..3198c18 --- /dev/null +++ b/gitautodeploy/wsserver.py @@ -0,0 +1,21 @@ +from SimpleWebSocketServer import WebSocket + +clients = [] +class WebSocketClientHandler(WebSocket): + + def handleMessage(self): + for client in clients: + if client != self: + client.sendMessage(self.address[0] + u' - ' + self.data) + + def handleConnected(self): + print (self.address, 'connected') + for client in clients: + client.sendMessage(self.address[0] + u' - connected') + clients.append(self) + + def handleClose(self): + clients.remove(self) + print (self.address, 'closed') + for client in clients: + client.sendMessage(self.address[0] + u' - disconnected') diff --git a/platforms/linux/initfiles/debianLSBInitScripts/git-auto-deploy b/platforms/linux/initfiles/debianLSBInitScripts/git-auto-deploy index 310cf07..2b357db 100755 --- a/platforms/linux/initfiles/debianLSBInitScripts/git-auto-deploy +++ b/platforms/linux/initfiles/debianLSBInitScripts/git-auto-deploy @@ -14,7 +14,7 @@ NAME="git-auto-deploy" PATH=/sbin:/usr/sbin:/bin:/usr/bin DESC="GitAutodeploy" -DAEMON="/usr/bin/env python /opt/Git-Auto-Deploy/gitautodeploy" +DAEMON="/usr/bin/env python -m /opt/Git-Auto-Deploy/gitautodeploy" DAEMON_UID=root DAEMON_GID=root RUNDIR=/var/run/$NAME diff --git a/platforms/linux/initfiles/systemd/git-auto-deploy.service b/platforms/linux/initfiles/systemd/git-auto-deploy.service index c73ff6c..1dd2ff0 100644 --- a/platforms/linux/initfiles/systemd/git-auto-deploy.service +++ b/platforms/linux/initfiles/systemd/git-auto-deploy.service @@ -5,7 +5,7 @@ Description=GitAutoDeploy User=www-data Group=www-data WorkingDirectory=/opt/Git-Auto-Deploy/ -ExecStart=/usr/bin/python /opt/Git-Auto-Deploy/gitautodeploy --daemon-mode +ExecStart=/usr/bin/python -m /opt/Git-Auto-Deploy/gitautodeploy --daemon-mode [Install] WantedBy=multi-user.target |