diff options
Diffstat (limited to 'gitautodeploy/gitautodeploy.py')
| -rw-r--r-- | gitautodeploy/gitautodeploy.py | 501 |
1 files changed, 191 insertions, 310 deletions
diff --git a/gitautodeploy/gitautodeploy.py b/gitautodeploy/gitautodeploy.py index c4f7ed8..8f3b172 100644 --- a/gitautodeploy/gitautodeploy.py +++ b/gitautodeploy/gitautodeploy.py @@ -15,10 +15,12 @@ class LogInterface(object): class GitAutoDeploy(object): _instance = None _server = None - _config = None + _config = {} + _port = None + _pid = None def __new__(cls, *args, **kwargs): - """Overload constructor to enable Singleton access""" + """Overload constructor to enable singleton access""" if not cls._instance: cls._instance = super(GitAutoDeploy, cls).__new__( cls, *args, **kwargs) @@ -26,21 +28,23 @@ class GitAutoDeploy(object): @staticmethod def debug_diagnosis(port): + """Display information about what process is using the specified port.""" import logging logger = logging.getLogger() pid = GitAutoDeploy.get_pid_on_port(port) if pid is False: - logger.warning('I don\'t know the number of pid that is using my configured port') + logger.warning('Unable to determine what PID is using port %s' % port) return - logger.info('Process with pid number %s is using port %s' % (pid, port)) + logger.info('Process with PID %s is using port %s' % (pid, port)) with open("/proc/%s/cmdline" % pid) as f: cmdline = f.readlines() - logger.info('cmdline ->', cmdline[0].replace('\x00', ' ')) + logger.info('Process with PID %s was started using the command: %s' % (pid, cmdline[0].replace('\x00', ' '))) @staticmethod def get_pid_on_port(port): + """Determine what process (PID) is using a specific port.""" import os with open("/proc/net/tcp", 'r') as f: @@ -78,163 +82,6 @@ class GitAutoDeploy(object): return mpid - def find_config_file_path(self): - """Attempt to find a config file in cwd and script path.""" - - import os - import re - import logging - logger = logging.getLogger() - - # Look for a custom config file if no path is provided as argument - target_directories = [ - os.path.dirname(os.path.realpath(__file__)), # Script path - ] - - # Add current CWD if not identical to script path - if not os.getcwd() in target_directories: - target_directories.append(os.getcwd()) - - target_directories.reverse() - - # Look for a *conf.json or *config.json - for dir in target_directories: - if not os.access(dir, os.R_OK): - continue - for item in os.listdir(dir): - if re.match(r".*conf(ig)?\.json$", item): - path = os.path.realpath(os.path.join(dir, item)) - logger.info("Using '%s' as config" % path) - return path - - return - - def read_json_file(self, file_path): - import json - import logging - import re - logger = logging.getLogger() - - try: - json_string = open(file_path).read() - - except Exception as e: - logger.critical("Could not load %s file\n" % file_path) - raise e - - try: - # Remove commens from JSON (makes sample config options easier) - regex = r'\s*(#|\/{2}).*$' - regex_inline = r'(:?(?:\s)*([A-Za-z\d\.{}]*)|((?<=\").*\"),?)(?:\s)*(((#|(\/{2})).*)|)$' - lines = json_string.split('\n') - - for index, line in enumerate(lines): - if re.search(regex, line): - if re.search(r'^' + regex, line, re.IGNORECASE): - lines[index] = "" - elif re.search(regex_inline, line): - lines[index] = re.sub(regex_inline, r'\1', line) - - data = json.loads('\n'.join(lines)) - - except Exception as e: - logger.critical("%s file is not valid JSON\n" % file_path) - raise e - - return data - - def read_repo_config_from_environment(self, config_data): - """Look for repository config in any defined environment variables. If - found, import to main config.""" - import logging - import os - - if 'GAD_REPO_URL' not in os.environ: - return config_data - - logger = logging.getLogger() - - repo_config = { - 'url': os.environ['GAD_REPO_URL'] - } - - logger.info("Added configuration for '%s' found in environment variables" % os.environ['GAD_REPO_URL']) - - if 'GAD_REPO_BRANCH' in os.environ: - repo_config['branch'] = os.environ['GAD_REPO_BRANCH'] - - if 'GAD_REPO_REMOTE' in os.environ: - repo_config['remote'] = os.environ['GAD_REPO_REMOTE'] - - if 'GAD_REPO_PATH' in os.environ: - repo_config['path'] = os.environ['GAD_REPO_PATH'] - - if 'GAD_REPO_DEPLOY' in os.environ: - repo_config['deploy'] = os.environ['GAD_REPO_DEPLOY'] - - if not 'repositories' in config_data: - config_data['repositories'] = [] - - config_data['repositories'].append(repo_config) - - return config_data - - def init_config(self, config_data): - import os - import re - import logging - logger = logging.getLogger() - - self._config = config_data - - # Translate any ~ in the path into /home/<user> - if 'pidfilepath' in self._config: - self._config['pidfilepath'] = os.path.expanduser(self._config['pidfilepath']) - - if 'repositories' not in self._config: - self._config['repositories'] = [] - - for repo_config in self._config['repositories']: - - # Setup branch if missing - if 'branch' not in repo_config: - repo_config['branch'] = "master" - - # Setup remote if missing - if 'remote' not in repo_config: - repo_config['remote'] = "origin" - - # Setup deploy commands list if not present - if 'deploy_commands' not in repo_config: - repo_config['deploy_commands'] = [] - - # Check if any global pre deploy commands is specified - if 'global_deploy' in self._config and len(self._config['global_deploy'][0]) is not 0: - repo_config['deploy_commands'].insert(0, self._config['global_deploy'][0]) - - # Check if any repo specific deploy command is specified - if 'deploy' in repo_config: - repo_config['deploy_commands'].append(repo_config['deploy']) - - # Check if any global post deploy command is specified - if 'global_deploy' in self._config and len(self._config['global_deploy'][1]) is not 0: - repo_config['deploy_commands'].append(self._config['global_deploy'][1]) - - # If a Bitbucket repository is configured using the https:// URL, a username is usually - # specified in the beginning of the URL. To be able to compare configured Bitbucket - # repositories with incoming web hook events, this username needs to be stripped away in a - # copy of the URL. - if 'url' in repo_config and 'bitbucket_username' not in repo_config: - regexp = re.search(r"^(https?://)([^@]+)@(bitbucket\.org/)(.+)$", repo_config['url']) - if regexp: - repo_config['url_without_usernme'] = regexp.group(1) + regexp.group(3) + regexp.group(4) - - # Translate any ~ in the path into /home/<user> - if 'path' in repo_config: - repo_config['path'] = os.path.expanduser(repo_config['path']) - - return self._config - def clone_all_repos(self): """Iterates over all configured repositories and clones them to their configured paths.""" @@ -244,22 +91,34 @@ class GitAutoDeploy(object): from wrappers import GitWrapper logger = logging.getLogger() + if not 'repositories' in self._config: + return + # Iterate over all configured repositories for repo_config in self._config['repositories']: - + + # Only clone repositories with a configured path + if 'url' not in repo_config: + logger.critical("Repository has no configured URL") + self.close() + self.exit() + return + # Only clone repositories with a configured path if 'path' not in repo_config: - logger.info("Repository %s will not be cloned (no path configured)" % repo_config['url']) + logger.debug("Repository %s will not be cloned (no path configured)" % repo_config['url']) continue - + if os.path.isdir(repo_config['path']) and os.path.isdir(repo_config['path']+'/.git'): - logger.info("Repository %s already present" % repo_config['url']) + logger.debug("Repository %s already present" % repo_config['url']) continue + logger.info("Repository %s not present and needs to be cloned" % repo_config['url']) + # Clone repository - GitWrapper.clone(url=repo_config['url'], branch=repo_config['branch'], path=repo_config['path']) - - if os.path.isdir(repo_config['path']): + ret = GitWrapper.clone(url=repo_config['url'], branch=repo_config['branch'], path=repo_config['path']) + + if ret == 0 and os.path.isdir(repo_config['path']): logger.info("Repository %s successfully cloned" % repo_config['url']) else: logger.error("Unable to clone %s branch of repository %s" % (repo_config['branch'], repo_config['url'])) @@ -289,19 +148,25 @@ class GitAutoDeploy(object): logger.error('Could not find regexp match in path: %s' % url) def kill_conflicting_processes(self): + """Attempt to kill any process already using the configured port.""" import os import logging + import signal logger = logging.getLogger() pid = GitAutoDeploy.get_pid_on_port(self._config['port']) if pid is False: - logger.error('[KILLER MODE] I don\'t know the number of pid ' + - 'that is using my configured port\n[KILLER MODE] ' + - 'Maybe no one? Please, use --force option carefully') + logger.warning('No process is currently using port %s.' % self._config['port']) return False - os.kill(pid, signal.SIGKILL) + if hasattr(signal, 'SIGKILL'): + os.kill(pid, signal.SIGKILL) + elif hasattr(signal, 'SIGHUP'): + os.kill(pid, signal.SIGHUP) + else: + os.kill(pid, 1) + return True def create_pid_file(self): @@ -316,14 +181,27 @@ class GitAutoDeploy(object): def remove_pid_file(self): import os - os.remove(self._config['pidfilepath']) + import errno + if 'pidfilepath' in self._config and self._config['pidfilepath']: + try: + os.remove(self._config['pidfilepath']) + except OSError, e: + if e.errno != errno.ENOENT: # errno.ENOENT = no such file or directory + raise - def exit(self): + def close(self): import sys import logging logger = logging.getLogger() logger.info('Goodbye') self.remove_pid_file() + if 'intercept-stdout' in self._config and self._config['intercept-stdout']: + sys.stdout = self._default_stdout + sys.stderr = self._default_stderr + + def exit(self): + import sys + self.close() sys.exit(0) @staticmethod @@ -358,152 +236,56 @@ class GitAutoDeploy(object): return 0 - def run(self): + def setup(self, config): + """Setup an instance of GAD based on the provided config object.""" import sys from BaseHTTPServer import HTTPServer import socket import os import logging - import argparse from lock import Lock from httpserver import WebhookRequestHandler - # Attempt to retrieve default config values from environment variables - default_quiet_value = 'GAD_QUIET' in os.environ or False - default_daemon_mode_value = 'GAD_DAEMON_MODE' in os.environ or False - default_config_value = 'GAD_CONFIG' in os.environ and os.environ['GAD_CONFIG'] or None - default_ssh_keygen_value = 'GAD_SSH_KEYGEN' in os.environ or False - default_force_value = 'GAD_FORCE' in os.environ or False - default_use_ssl = 'GAD_SSL' in os.environ or False - default_ssl_pem_file_path = 'GAD_SSL_PEM_FILE' in os.environ and os.environ['GAD_SSL_PEM_FILE'] or '~/.gitautodeploy.pem' - default_pid_file_value = 'GAD_PID_FILE' in os.environ and os.environ['GAD_PID_FILE'] or '~/.gitautodeploy.pid' - default_log_file_value = 'GAD_LOG_FILE' in os.environ and os.environ['GAD_LOG_FILE'] or None - default_host_value = 'GAD_HOST' in os.environ and os.environ['GAD_HOST'] or '0.0.0.0' - default_port_value = 'GAD_PORT' in os.environ and int(os.environ['GAD_PORT']) or 8001 - - parser = argparse.ArgumentParser() - - parser.add_argument("-d", "--daemon-mode", - help="run in background (daemon mode)", - default=default_daemon_mode_value, - action="store_true") - - parser.add_argument("-q", "--quiet", - help="supress console output", - default=default_quiet_value, - action="store_true") - - parser.add_argument("-c", "--config", - help="custom configuration file", - default=default_config_value, - type=str) - - parser.add_argument("--ssh-keygen", - help="scan repository hosts for ssh keys", - default=default_ssh_keygen_value, - action="store_true") - - parser.add_argument("--force", - help="kill any process using the configured port", - default=default_force_value, - action="store_true") - - parser.add_argument("--pid-file", - help="specify a custom pid file", - default=default_pid_file_value, - type=str) - - parser.add_argument("--log-file", - help="specify a log file", - default=default_log_file_value, - type=str) - - parser.add_argument("--host", - help="address to bind to", - default=default_host_value, - type=str) - - parser.add_argument("--port", - help="port to bind to", - default=default_port_value, - type=int) - - parser.add_argument("--ssl", - help="use ssl", - default=default_use_ssl, - action="store_true") - - parser.add_argument("--ssl-pem", - help="path to ssl pem file", - default=default_ssl_pem_file_path, - type=str) - - args = parser.parse_args() + # Attatch config values to this instance + self._config = config # Set up logging logger = logging.getLogger() logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") # Enable console output? - if args.quiet: + if ('quiet' in self._config and self._config['quiet']) or ('daemon-mode' in self._config and self._config['daemon-mode']): logger.addHandler(logging.NullHandler()) else: consoleHandler = logging.StreamHandler() consoleHandler.setFormatter(logFormatter) - logger.addHandler(consoleHandler) - - # All logs are recording - logger.setLevel(logging.NOTSET) - - # Look for log file path provided in argument - config_file_path = None - if args.config: - config_file_path = os.path.realpath(args.config) - logger.info('Using custom configuration file \'%s\'' % config_file_path) - - # Try to find a config file on the file system - if not config_file_path: - config_file_path = self.find_config_file_path() - - # Read config data from json file - if config_file_path: - config_data = self.read_json_file(config_file_path) - else: - logger.info('No configuration file found or specified. Using default values.') - config_data = {} - - # Configuration options coming from environment or command line will - # override those coming from config file - if args.pid_file: - config_data['pidfilepath'] = args.pid_file - - if args.log_file: - config_data['logfilepath'] = args.log_file - - if args.host: - config_data['host'] = args.host - if args.port: - config_data['port'] = args.port + # Check if a stream handler is already present (will be if GAD is started by test script) + handler_present = False + for handler in logger.handlers: + if isinstance(handler, type(consoleHandler)): + handler_present = True + break - # Extend config data with any repository defined by environment variables - config_data = self.read_repo_config_from_environment(config_data) + if not handler_present: + logger.addHandler(consoleHandler) - # Initialize config using config file data - self.init_config(config_data) + # Set logging level + if 'log-level' in self._config: + level = logging.getLevelName(self._config['log-level']) + logger.setLevel(level) - # Translate any ~ in the path into /home/<user> - if 'logfilepath' in self._config: - log_file_path = os.path.expanduser(self._config['logfilepath']) - fileHandler = logging.FileHandler(log_file_path) + if 'logfilepath' in self._config and self._config['logfilepath']: + # Translate any ~ in the path into /home/<user> + fileHandler = logging.FileHandler(self._config['logfilepath']) fileHandler.setFormatter(logFormatter) logger.addHandler(fileHandler) - if args.ssh_keygen: + if 'ssh-keygen' in self._config and self._config['ssh-keygen']: logger.info('Scanning repository hosts for ssh keys...') self.ssh_key_scan() - if args.force: + if 'force' in self._config and self._config['force']: logger.info('Attempting to kill any other process currently occupying port %s' % self._config['port']) self.kill_conflicting_processes() @@ -512,15 +294,19 @@ class GitAutoDeploy(object): # Set default stdout and stderr to our logging interface (that writes # to file and console depending on user preference) - sys.stdout = LogInterface(logger.info) - sys.stderr = LogInterface(logger.error) + if 'intercept-stdout' in self._config and self._config['intercept-stdout']: + self._default_stdout = sys.stdout + self._default_stderr = sys.stderr + sys.stdout = LogInterface(logger.info) + sys.stderr = LogInterface(logger.error) - if args.daemon_mode: + if 'daemon-mode' in self._config and self._config['daemon-mode']: logger.info('Starting Git Auto Deploy in daemon mode') GitAutoDeploy.create_daemon() else: logger.info('Git Auto Deploy started') + self._pid = os.getpid() self.create_pid_file() # Clear any existing lock files, with no regard to possible ongoing processes @@ -536,27 +322,72 @@ class GitAutoDeploy(object): self._server = HTTPServer((self._config['host'], self._config['port']), WebhookRequestHandler) - if args.ssl: + + if 'ssl' in self._config and self._config['ssl']: import ssl logger.info("enabling ssl") self._server.socket = ssl.wrap_socket(self._server.socket, - certfile=os.path.expanduser(args.ssl_pem), + certfile=os.path.expanduser(self._config['ssl-pem']), server_side=True) sa = self._server.socket.getsockname() logger.info("Listening on %s port %s", sa[0], sa[1]) + + # Actual port bound to (nessecary when OS picks randomly free port) + self._port = sa[1] + + except socket.error, e: + + logger.critical("Error on socket: %s" % e) + GitAutoDeploy.debug_diagnosis(self._config['port']) + + sys.exit(1) + + def serve_forever(self): + """Start listening for incoming requests.""" + import sys + import socket + import logging + + # Set up logging + logger = logging.getLogger() + + try: self._server.serve_forever() except socket.error, e: + logger.critical("Error on socket: %s" % e) + sys.exit(1) + + except KeyboardInterrupt, e: + logger.info('Requested close by keyboard interrupt signal') + self.stop() + self.exit() + + def handle_request(self): + """Start listening for incoming requests.""" + import sys + import socket + import logging - if not args.daemon_mode: - logger.critical("Error on socket: %s" % e) - GitAutoDeploy.debug_diagnosis(self._config['port']) + # Set up logging + logger = logging.getLogger() + + try: + self._server.handle_request() + except socket.error, e: + logger.critical("Error on socket: %s" % e) sys.exit(1) + + except KeyboardInterrupt, e: + logger.info('Requested close by keyboard interrupt signal') + self.stop() + self.exit() def stop(self): - if self._server is not None: - self._server.socket.close() + if self._server is None: + return + self._server.socket.close() def signal_handler(self, signum, frame): import logging @@ -564,7 +395,8 @@ class GitAutoDeploy(object): self.stop() if signum == 1: - self.run() + self.setup(self._config) + self.serve_forever() return elif signum == 2: @@ -575,16 +407,65 @@ class GitAutoDeploy(object): self.exit() - def main(): import signal from gitautodeploy import GitAutoDeploy + from cli.config import get_config_defaults, get_config_from_environment, get_config_from_argv, find_config_file, get_config_from_file, get_repo_config_from_environment, init_config + import sys + import os app = GitAutoDeploy() - signal.signal(signal.SIGHUP, app.signal_handler) - signal.signal(signal.SIGINT, app.signal_handler) - signal.signal(signal.SIGABRT, app.signal_handler) - signal.signal(signal.SIGPIPE, signal.SIG_IGN) + if hasattr(signal, 'SIGHUP'): + signal.signal(signal.SIGHUP, app.signal_handler) + if hasattr(signal, 'SIGINT'): + signal.signal(signal.SIGINT, app.signal_handler) + if hasattr(signal, 'SIGABRT'): + signal.signal(signal.SIGABRT, app.signal_handler) + if hasattr(signal, 'SIGPIPE') and hasattr(signal, 'SIG_IGN'): + signal.signal(signal.SIGPIPE, signal.SIG_IGN) + + config = get_config_defaults() + + # Get config values from environment variables and commadn line arguments + environment_config = get_config_from_environment() + argv_config = get_config_from_argv(sys.argv[1:]) + + # Merge config values + config.update(environment_config) + config.update(argv_config) + + # Config file path provided? + if 'config' in config and config['config']: + config_file_path = os.path.realpath(config['config']) + + else: + + # Directories to scan for config files + target_directories = [ + os.getcwd(), # cwd + os.path.dirname(os.path.realpath(__file__)) # script path + ] + + config_file_path = find_config_file(target_directories) + + # Config file path provided or found? + if config_file_path: + file_config = get_config_from_file(config_file_path) + config.update(file_config) + + # Extend config data with any repository defined by environment variables + repo_config = get_repo_config_from_environment() + + if repo_config: + + if not 'repositories' in config: + config['repositories'] = [] + + config['repositories'].append(repo_config) + + # Initialize config by expanding with missing values + init_config(config) - app.run()
\ No newline at end of file + app.setup(config) + app.serve_forever() |