Raphaël G. Git Repositories - youtubedl/blob - devscripts/buildserver.py

   1 #!/usr/bin/python3
   2
   3 from http.server import HTTPServer, BaseHTTPRequestHandler
   4 from socketserver import ThreadingMixIn
   5 import argparse
   6 import ctypes
   7 import functools
   8 import sys
   9 import threading
  10 import traceback
  11 import os.path
  12
  13
  14 class BuildHTTPServer(ThreadingMixIn, HTTPServer):
  15     allow_reuse_address = True
  16
  17
  18 advapi32 = ctypes.windll.advapi32
  19
  20 SC_MANAGER_ALL_ACCESS = 0xf003f
  21 SC_MANAGER_CREATE_SERVICE = 0x02
  22 SERVICE_WIN32_OWN_PROCESS = 0x10
  23 SERVICE_AUTO_START = 0x2
  24 SERVICE_ERROR_NORMAL = 0x1
  25 DELETE = 0x00010000
  26 SERVICE_STATUS_START_PENDING = 0x00000002
  27 SERVICE_STATUS_RUNNING = 0x00000004
  28 SERVICE_ACCEPT_STOP = 0x1
  29
  30 SVCNAME = 'youtubedl_builder'
  31
  32 LPTSTR = ctypes.c_wchar_p
  33 START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
  34
  35
  36 class SERVICE_TABLE_ENTRY(ctypes.Structure):
  37     _fields_ = [
  38         ('lpServiceName', LPTSTR),
  39         ('lpServiceProc', START_CALLBACK)
  40     ]
  41
  42
  43 HandlerEx = ctypes.WINFUNCTYPE(
  44     ctypes.c_int,     # return
  45     ctypes.c_int,     # dwControl
  46     ctypes.c_int,     # dwEventType
  47     ctypes.c_void_p,  # lpEventData,
  48     ctypes.c_void_p,  # lpContext,
  49 )
  50
  51
  52 def _ctypes_array(c_type, py_array):
  53     ar = (c_type * len(py_array))()
  54     ar[:] = py_array
  55     return ar
  56
  57
  58 def win_OpenSCManager():
  59     res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
  60     if not res:
  61         raise Exception('Opening service manager failed - '
  62                         'are you running this as administrator?')
  63     return res
  64
  65
  66 def win_install_service(service_name, cmdline):
  67     manager = win_OpenSCManager()
  68     try:
  69         h = advapi32.CreateServiceW(
  70             manager, service_name, None,
  71             SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
  72             SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
  73             cmdline, None, None, None, None, None)
  74         if not h:
  75             raise OSError('Service creation failed: %s' % ctypes.FormatError())
  76
  77         advapi32.CloseServiceHandle(h)
  78     finally:
  79         advapi32.CloseServiceHandle(manager)
  80
  81
  82 def win_uninstall_service(service_name):
  83     manager = win_OpenSCManager()
  84     try:
  85         h = advapi32.OpenServiceW(manager, service_name, DELETE)
  86         if not h:
  87             raise OSError('Could not find service %s: %s' % (
  88                 service_name, ctypes.FormatError()))
  89
  90         try:
  91             if not advapi32.DeleteService(h):
  92                 raise OSError('Deletion failed: %s' % ctypes.FormatError())
  93         finally:
  94             advapi32.CloseServiceHandle(h)
  95     finally:
  96         advapi32.CloseServiceHandle(manager)
  97
  98
  99 def win_service_report_event(service_name, msg, is_error=True):
 100     with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
 101         f.write(msg + '\n')
 102
 103     event_log = advapi32.RegisterEventSourceW(None, service_name)
 104     if not event_log:
 105         raise OSError('Could not report event: %s' % ctypes.FormatError())
 106
 107     try:
 108         type_id = 0x0001 if is_error else 0x0004
 109         event_id = 0xc0000000 if is_error else 0x40000000
 110         lines = _ctypes_array(LPTSTR, [msg])
 111
 112         if not advapi32.ReportEventW(
 113                 event_log, type_id, 0, event_id, None, len(lines), 0,
 114                 lines, None):
 115             raise OSError('Event reporting failed: %s' % ctypes.FormatError())
 116     finally:
 117         advapi32.DeregisterEventSource(event_log)
 118
 119
 120 def win_service_handler(stop_event, *args):
 121     try:
 122         raise ValueError('Handler called with args ' + repr(args))
 123         TODO
 124     except Exception as e:
 125         tb = traceback.format_exc()
 126         msg = str(e) + '\n' + tb
 127         win_service_report_event(service_name, msg, is_error=True)
 128         raise
 129
 130
 131 def win_service_set_status(handle, status_code):
 132     svcStatus = SERVICE_STATUS()
 133     svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
 134     svcStatus.dwCurrentState = status_code
 135     svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
 136
 137     svcStatus.dwServiceSpecificExitCode = 0
 138
 139     if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
 140         raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
 141
 142
 143 def win_service_main(service_name, real_main, argc, argv_raw):
 144     try:
 145         #args = [argv_raw[i].value for i in range(argc)]
 146         stop_event = threading.Event()
 147         handler = HandlerEx(functools.partial(stop_event, win_service_handler))
 148         h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
 149         if not h:
 150             raise OSError('Handler registration failed: %s' %
 151                           ctypes.FormatError())
 152
 153         TODO
 154     except Exception as e:
 155         tb = traceback.format_exc()
 156         msg = str(e) + '\n' + tb
 157         win_service_report_event(service_name, msg, is_error=True)
 158         raise
 159
 160
 161 def win_service_start(service_name, real_main):
 162     try:
 163         cb = START_CALLBACK(
 164             functools.partial(win_service_main, service_name, real_main))
 165         dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
 166             SERVICE_TABLE_ENTRY(
 167                 service_name,
 168                 cb
 169             ),
 170             SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
 171         ])
 172
 173         if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
 174             raise OSError('ctypes start failed: %s' % ctypes.FormatError())
 175     except Exception as e:
 176         tb = traceback.format_exc()
 177         msg = str(e) + '\n' + tb
 178         win_service_report_event(service_name, msg, is_error=True)
 179         raise
 180
 181
 182 def main(args=None):
 183     parser = argparse.ArgumentParser()
 184     parser.add_argument('-i', '--install',
 185                         action='store_const', dest='action', const='install',
 186                         help='Launch at Windows startup')
 187     parser.add_argument('-u', '--uninstall',
 188                         action='store_const', dest='action', const='uninstall',
 189                         help='Remove Windows service')
 190     parser.add_argument('-s', '--service',
 191                         action='store_const', dest='action', const='service',
 192                         help='Run as a Windows service')
 193     parser.add_argument('-b', '--bind', metavar='<host:port>',
 194                         action='store', default='localhost:8142',
 195                         help='Bind to host:port (default %default)')
 196     options = parser.parse_args(args=args)
 197
 198     if options.action == 'install':
 199         fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
 200         cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
 201         win_install_service(SVCNAME, cmdline)
 202         return
 203
 204     if options.action == 'uninstall':
 205         win_uninstall_service(SVCNAME)
 206         return
 207
 208     if options.action == 'service':
 209         win_service_start(SVCNAME, main)
 210         return
 211
 212     host, port_str = options.bind.split(':')
 213     port = int(port_str)
 214
 215     print('Listening on %s:%d' % (host, port))
 216     srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
 217     thr = threading.Thread(target=srv.serve_forever)
 218     thr.start()
 219     input('Press ENTER to shut down')
 220     srv.shutdown()
 221     thr.join()
 222
 223
 224 def rmtree(path):
 225     for name in os.listdir(path):
 226         fname = os.path.join(path, name)
 227         if os.path.isdir(fname):
 228             rmtree(fname)
 229         else:
 230             os.chmod(fname, 0o666)
 231             os.remove(fname)
 232     os.rmdir(path)
 233
 234 #==============================================================================
 235
 236 class BuildError(Exception):
 237     def __init__(self, output, code=500):
 238         self.output = output
 239         self.code = code
 240
 241     def __str__(self):
 242         return self.output
 243
 244
 245 class HTTPError(BuildError):
 246     pass
 247
 248
 249 class PythonBuilder(object):
 250     def __init__(self, **kwargs):
 251         pythonVersion = kwargs.pop('python', '2.7')
 252         try:
 253             key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % pythonVersion)
 254             try:
 255                 self.pythonPath, _ = _winreg.QueryValueEx(key, '')
 256             finally:
 257                 _winreg.CloseKey(key)
 258         except Exception:
 259             raise BuildError('No such Python version: %s' % pythonVersion)
 260
 261         super(PythonBuilder, self).__init__(**kwargs)
 262
 263
 264 class GITInfoBuilder(object):
 265     def __init__(self, **kwargs):
 266         try:
 267             self.user, self.repoName = kwargs['path'][:2]
 268             self.rev = kwargs.pop('rev')
 269         except ValueError:
 270             raise BuildError('Invalid path')
 271         except KeyError as e:
 272             raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
 273
 274         path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
 275         if not os.path.exists(path):
 276             os.makedirs(path)
 277         self.basePath = tempfile.mkdtemp(dir=path)
 278         self.buildPath = os.path.join(self.basePath, 'build')
 279
 280         super(GITInfoBuilder, self).__init__(**kwargs)
 281
 282
 283 class GITBuilder(GITInfoBuilder):
 284     def build(self):
 285         try:
 286             subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
 287             subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
 288         except subprocess.CalledProcessError as e:
 289             raise BuildError(e.output)
 290
 291         super(GITBuilder, self).build()
 292
 293
 294 class YoutubeDLBuilder(object):
 295     authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile']
 296
 297     def __init__(self, **kwargs):
 298         if self.repoName != 'youtube-dl':
 299             raise BuildError('Invalid repository "%s"' % self.repoName)
 300         if self.user not in self.authorizedUsers:
 301             raise HTTPError('Unauthorized user "%s"' % self.user, 401)
 302
 303         super(YoutubeDLBuilder, self).__init__(**kwargs)
 304
 305     def build(self):
 306         try:
 307             subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
 308                                     cwd=self.buildPath)
 309         except subprocess.CalledProcessError as e:
 310             raise BuildError(e.output)
 311
 312         super(YoutubeDLBuilder, self).build()
 313
 314
 315 class DownloadBuilder(object):
 316     def __init__(self, **kwargs):
 317         self.handler = kwargs.pop('handler')
 318         self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
 319         self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
 320         if not self.srcPath.startswith(self.buildPath):
 321             raise HTTPError(self.srcPath, 401)
 322
 323         super(DownloadBuilder, self).__init__(**kwargs)
 324
 325     def build(self):
 326         if not os.path.exists(self.srcPath):
 327             raise HTTPError('No such file', 404)
 328         if os.path.isdir(self.srcPath):
 329             raise HTTPError('Is a directory: %s' % self.srcPath, 401)
 330
 331         self.handler.send_response(200)
 332         self.handler.send_header('Content-Type', 'application/octet-stream')
 333         self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
 334         self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
 335         self.handler.end_headers()
 336
 337         with open(self.srcPath, 'rb') as src:
 338             shutil.copyfileobj(src, self.handler.wfile)
 339
 340         super(DownloadBuilder, self).build()
 341
 342
 343 class CleanupTempDir(object):
 344     def build(self):
 345         try:
 346             rmtree(self.basePath)
 347         except Exception as e:
 348             print('WARNING deleting "%s": %s' % (self.basePath, e))
 349
 350         super(CleanupTempDir, self).build()
 351
 352
 353 class Null(object):
 354     def __init__(self, **kwargs):
 355         pass
 356
 357     def start(self):
 358         pass
 359
 360     def close(self):
 361         pass
 362
 363     def build(self):
 364         pass
 365
 366
 367 class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
 368     pass
 369
 370
 371 class BuildHTTPRequestHandler(BaseHTTPRequestHandler):
 372     actionDict = { 'build': Builder, 'download': Builder } # They're the same, no more caching.
 373
 374     def do_GET(self):
 375         path = urlparse.urlparse(self.path)
 376         paramDict = dict([(key, value[0]) for key, value in urlparse.parse_qs(path.query).items()])
 377         action, _, path = path.path.strip('/').partition('/')
 378         if path:
 379             path = path.split('/')
 380             if action in self.actionDict:
 381                 try:
 382                     builder = self.actionDict[action](path=path, handler=self, **paramDict)
 383                     builder.start()
 384                     try:
 385                         builder.build()
 386                     finally:
 387                         builder.close()
 388                 except BuildError as e:
 389                     self.send_response(e.code)
 390                     msg = unicode(e).encode('UTF-8')
 391                     self.send_header('Content-Type', 'text/plain; charset=UTF-8')
 392                     self.send_header('Content-Length', len(msg))
 393                     self.end_headers()
 394                     self.wfile.write(msg)
 395                 except HTTPError as e:
 396                     self.send_response(e.code, str(e))
 397             else:
 398                 self.send_response(500, 'Unknown build method "%s"' % action)
 399         else:
 400             self.send_response(500, 'Malformed URL')
 401
 402 #==============================================================================
 403
 404 if __name__ == '__main__':
 405     main()