diff options
Diffstat (limited to 'import-layers/yocto-poky/bitbake/lib/bb/server')
3 files changed, 757 insertions, 0 deletions
diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py b/import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py new file mode 100644 index 000000000..538a633fe --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/__init__.py @@ -0,0 +1,99 @@ +# +# BitBake Base Server Code +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2008 Richard Purdie +# Copyright (C) 2013 Alexandru Damian +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" Base code for Bitbake server process + +Have a common base for that all Bitbake server classes ensures a consistent +approach to the interface, and minimize risks associated with code duplication. + +""" + +""" BaseImplServer() the base class for all XXServer() implementations. + + These classes contain the actual code that runs the server side, i.e. + listens for the commands and executes them. Although these implementations + contain all the data of the original bitbake command, i.e the cooker instance, + they may well run on a different process or even machine. + +""" + +class BaseImplServer(): + def __init__(self): + self._idlefuns = {} + + def addcooker(self, cooker): + self.cooker = cooker + + def register_idle_function(self, function, data): + """Register a function to be called while the server is idle""" + assert hasattr(function, '__call__') + self._idlefuns[function] = data + + + +""" BitBakeBaseServerConnection class is the common ancestor to all + BitBakeServerConnection classes. + + These classes control the remote server. The only command currently + implemented is the terminate() command. + +""" + +class BitBakeBaseServerConnection(): + def __init__(self, serverImpl): + pass + + def terminate(self): + pass + + def setupEventQueue(self): + pass + + +""" BitBakeBaseServer class is the common ancestor to all Bitbake servers + + Derive this class in order to implement a BitBakeServer which is the + controlling stub for the actual server implementation + +""" +class BitBakeBaseServer(object): + def initServer(self): + self.serverImpl = None # we ensure a runtime crash if not overloaded + self.connection = None + return + + def addcooker(self, cooker): + self.cooker = cooker + self.serverImpl.addcooker(cooker) + + def getServerIdleCB(self): + return self.serverImpl.register_idle_function + + def saveConnectionDetails(self): + return + + def detach(self): + return + + def establishConnection(self, featureset): + raise "Must redefine the %s.establishConnection()" % self.__class__.__name__ + + def endSession(self): + self.connection.terminate() diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/process.py b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py new file mode 100644 index 000000000..a3078a873 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/process.py @@ -0,0 +1,268 @@ +# +# BitBake Process based server. +# +# Copyright (C) 2010 Bob Foerster <robert@erafx.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements a multiprocessing.Process based server for bitbake. +""" + +import bb +import bb.event +import itertools +import logging +import multiprocessing +import os +import signal +import sys +import time +import select +from Queue import Empty +from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager + +from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer + +logger = logging.getLogger('BitBake') + +class ServerCommunicator(): + def __init__(self, connection, event_handle, server): + self.connection = connection + self.event_handle = event_handle + self.server = server + + def runCommand(self, command): + # @todo try/except + self.connection.send(command) + + if not self.server.is_alive(): + raise SystemExit + + while True: + # don't let the user ctrl-c while we're waiting for a response + try: + for idx in range(0,4): # 0, 1, 2, 3 + if self.connection.poll(5): + return self.connection.recv() + else: + bb.warn("Timeout while attempting to communicate with bitbake server") + bb.fatal("Gave up; Too many tries: timeout while attempting to communicate with bitbake server") + except KeyboardInterrupt: + pass + + def getEventHandle(self): + return self.event_handle.value + +class EventAdapter(): + """ + Adapter to wrap our event queue since the caller (bb.event) expects to + call a send() method, but our actual queue only has put() + """ + def __init__(self, queue): + self.queue = queue + + def send(self, event): + try: + self.queue.put(event) + except Exception as err: + print("EventAdapter puked: %s" % str(err)) + + +class ProcessServer(Process, BaseImplServer): + profile_filename = "profile.log" + profile_processed_filename = "profile.log.processed" + + def __init__(self, command_channel, event_queue, featurelist): + BaseImplServer.__init__(self) + Process.__init__(self) + self.command_channel = command_channel + self.event_queue = event_queue + self.event = EventAdapter(event_queue) + self.featurelist = featurelist + self.quit = False + + self.quitin, self.quitout = Pipe() + self.event_handle = multiprocessing.Value("i") + + def run(self): + for event in bb.event.ui_queue: + self.event_queue.put(event) + self.event_handle.value = bb.event.register_UIHhandler(self, True) + + bb.cooker.server_main(self.cooker, self.main) + + def main(self): + # Ignore SIGINT within the server, as all SIGINT handling is done by + # the UI and communicated to us + self.quitin.close() + signal.signal(signal.SIGINT, signal.SIG_IGN) + bb.utils.set_process_name("Cooker") + while not self.quit: + try: + if self.command_channel.poll(): + command = self.command_channel.recv() + self.runCommand(command) + if self.quitout.poll(): + self.quitout.recv() + self.quit = True + try: + self.runCommand(["stateForceShutdown"]) + except: + pass + + self.idle_commands(.1, [self.command_channel, self.quitout]) + except Exception: + logger.exception('Running command %s', command) + + self.event_queue.close() + bb.event.unregister_UIHhandler(self.event_handle.value) + self.command_channel.close() + self.cooker.shutdown(True) + self.quitout.close() + + def idle_commands(self, delay, fds=None): + nextsleep = delay + if not fds: + fds = [] + + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, False) + if retval is False: + del self._idlefuns[function] + nextsleep = None + elif retval is True: + nextsleep = None + elif isinstance(retval, float): + if (retval < nextsleep): + nextsleep = retval + elif nextsleep is None: + continue + else: + fds = fds + retval + except SystemExit: + raise + except Exception as exc: + if not isinstance(exc, bb.BBHandledException): + logger.exception('Running idle function') + del self._idlefuns[function] + self.quit = True + + if nextsleep is not None: + select.select(fds,[],[],nextsleep) + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + self.command_channel.send(self.cooker.command.runCommand(command)) + + def stop(self): + self.quitin.send("quit") + self.quitin.close() + +class BitBakeProcessServerConnection(BitBakeBaseServerConnection): + def __init__(self, serverImpl, ui_channel, event_queue): + self.procserver = serverImpl + self.ui_channel = ui_channel + self.event_queue = event_queue + self.connection = ServerCommunicator(self.ui_channel, self.procserver.event_handle, self.procserver) + self.events = self.event_queue + self.terminated = False + + def sigterm_terminate(self): + bb.error("UI received SIGTERM") + self.terminate() + + def terminate(self): + if self.terminated: + return + self.terminated = True + def flushevents(): + while True: + try: + event = self.event_queue.get(block=False) + except (Empty, IOError): + break + if isinstance(event, logging.LogRecord): + logger.handle(event) + + signal.signal(signal.SIGINT, signal.SIG_IGN) + self.procserver.stop() + + while self.procserver.is_alive(): + flushevents() + self.procserver.join(0.1) + + self.ui_channel.close() + self.event_queue.close() + self.event_queue.setexit() + +# Wrap Queue to provide API which isn't server implementation specific +class ProcessEventQueue(multiprocessing.queues.Queue): + def __init__(self, maxsize): + multiprocessing.queues.Queue.__init__(self, maxsize) + self.exit = False + bb.utils.set_process_name("ProcessEQueue") + + def setexit(self): + self.exit = True + + def waitEvent(self, timeout): + if self.exit: + sys.exit(1) + try: + if not self.server.is_alive(): + self.setexit() + return None + return self.get(True, timeout) + except Empty: + return None + + def getEvent(self): + try: + if not self.server.is_alive(): + self.setexit() + return None + return self.get(False) + except Empty: + return None + + +class BitBakeServer(BitBakeBaseServer): + def initServer(self, single_use=True): + # establish communication channels. We use bidirectional pipes for + # ui <--> server command/response pairs + # and a queue for server -> ui event notifications + # + self.ui_channel, self.server_channel = Pipe() + self.event_queue = ProcessEventQueue(0) + self.serverImpl = ProcessServer(self.server_channel, self.event_queue, None) + self.event_queue.server = self.serverImpl + + def detach(self): + self.serverImpl.start() + return + + def establishConnection(self, featureset): + + self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue) + + _, error = self.connection.connection.runCommand(["setFeatures", featureset]) + if error: + logger.error("Unable to set the cooker to the correct featureset: %s" % error) + raise BaseException(error) + signal.signal(signal.SIGTERM, lambda i, s: self.connection.sigterm_terminate()) + return self.connection diff --git a/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py b/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py new file mode 100644 index 000000000..ace1cf646 --- /dev/null +++ b/import-layers/yocto-poky/bitbake/lib/bb/server/xmlrpc.py @@ -0,0 +1,390 @@ +# +# BitBake XMLRPC Server +# +# Copyright (C) 2006 - 2007 Michael 'Mickey' Lauer +# Copyright (C) 2006 - 2008 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +""" + This module implements an xmlrpc server for BitBake. + + Use this by deriving a class from BitBakeXMLRPCServer and then adding + methods which you want to "export" via XMLRPC. If the methods have the + prefix xmlrpc_, then registering those function will happen automatically, + if not, you need to call register_function. + + Use register_idle_function() to add a function which the xmlrpc server + calls from within server_forever when no requests are pending. Make sure + that those functions are non-blocking or else you will introduce latency + in the server's main loop. +""" + +import bb +import xmlrpclib, sys +from bb import daemonize +from bb.ui import uievent +import hashlib, time +import socket +import os, signal +import threading +try: + import cPickle as pickle +except ImportError: + import pickle + +DEBUG = False + +from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler +import inspect, select, httplib + +from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer + +class BBTransport(xmlrpclib.Transport): + def __init__(self, timeout): + self.timeout = timeout + self.connection_token = None + xmlrpclib.Transport.__init__(self) + + # Modified from default to pass timeout to HTTPConnection + def make_connection(self, host): + #return an existing connection if possible. This allows + #HTTP/1.1 keep-alive. + if self._connection and host == self._connection[0]: + return self._connection[1] + + # create a HTTP connection object from a host descriptor + chost, self._extra_headers, x509 = self.get_host_info(host) + #store the host argument along with the connection object + self._connection = host, httplib.HTTPConnection(chost, timeout=self.timeout) + return self._connection[1] + + def set_connection_token(self, token): + self.connection_token = token + + def send_content(self, h, body): + if self.connection_token: + h.putheader("Bitbake-token", self.connection_token) + xmlrpclib.Transport.send_content(self, h, body) + +def _create_server(host, port, timeout = 60): + t = BBTransport(timeout) + s = xmlrpclib.ServerProxy("http://%s:%d/" % (host, port), transport=t, allow_none=True) + return s, t + +class BitBakeServerCommands(): + + def __init__(self, server): + self.server = server + self.has_client = False + + def registerEventHandler(self, host, port): + """ + Register a remote UI Event Handler + """ + s, t = _create_server(host, port) + + # we don't allow connections if the cooker is running + if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]): + return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.cooker.state) + + self.event_handle = bb.event.register_UIHhandler(s, True) + return self.event_handle, 'OK' + + def unregisterEventHandler(self, handlerNum): + """ + Unregister a remote UI Event Handler + """ + return bb.event.unregister_UIHhandler(handlerNum) + + def runCommand(self, command): + """ + Run a cooker command on the server + """ + return self.cooker.command.runCommand(command, self.server.readonly) + + def getEventHandle(self): + return self.event_handle + + def terminateServer(self): + """ + Trigger the server to quit + """ + self.server.quit = True + print("Server (cooker) exiting") + return + + def addClient(self): + if self.has_client: + return None + token = hashlib.md5(str(time.time())).hexdigest() + self.server.set_connection_token(token) + self.has_client = True + return token + + def removeClient(self): + if self.has_client: + self.server.set_connection_token(None) + self.has_client = False + if self.server.single_use: + self.server.quit = True + +# This request handler checks if the request has a "Bitbake-token" header +# field (this comes from the client side) and compares it with its internal +# "Bitbake-token" field (this comes from the server). If the two are not +# equal, it is assumed that a client is trying to connect to the server +# while another client is connected to the server. In this case, a 503 error +# ("service unavailable") is returned to the client. +class BitBakeXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): + def __init__(self, request, client_address, server): + self.server = server + SimpleXMLRPCRequestHandler.__init__(self, request, client_address, server) + + def do_POST(self): + try: + remote_token = self.headers["Bitbake-token"] + except: + remote_token = None + if remote_token != self.server.connection_token and remote_token != "observer": + self.report_503() + else: + if remote_token == "observer": + self.server.readonly = True + else: + self.server.readonly = False + SimpleXMLRPCRequestHandler.do_POST(self) + + def report_503(self): + self.send_response(503) + response = 'No more client allowed' + self.send_header("Content-type", "text/plain") + self.send_header("Content-length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + +class XMLRPCProxyServer(BaseImplServer): + """ not a real working server, but a stub for a proxy server connection + + """ + def __init__(self, host, port): + self.host = host + self.port = port + +class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer): + # remove this when you're done with debugging + # allow_reuse_address = True + + def __init__(self, interface, single_use=False): + """ + Constructor + """ + BaseImplServer.__init__(self) + self.single_use = single_use + # Use auto port configuration + if (interface[1] == -1): + interface = (interface[0], 0) + SimpleXMLRPCServer.__init__(self, interface, + requestHandler=BitBakeXMLRPCRequestHandler, + logRequests=False, allow_none=True) + self.host, self.port = self.socket.getsockname() + self.connection_token = None + #self.register_introspection_functions() + self.commands = BitBakeServerCommands(self) + self.autoregister_all_functions(self.commands, "") + self.interface = interface + + def addcooker(self, cooker): + BaseImplServer.addcooker(self, cooker) + self.commands.cooker = cooker + + def autoregister_all_functions(self, context, prefix): + """ + Convenience method for registering all functions in the scope + of this class that start with a common prefix + """ + methodlist = inspect.getmembers(context, inspect.ismethod) + for name, method in methodlist: + if name.startswith(prefix): + self.register_function(method, name[len(prefix):]) + + + def serve_forever(self): + # Start the actual XMLRPC server + bb.cooker.server_main(self.cooker, self._serve_forever) + + def _serve_forever(self): + """ + Serve Requests. Overloaded to honor a quit command + """ + self.quit = False + while not self.quit: + fds = [self] + nextsleep = 0.1 + for function, data in self._idlefuns.items(): + retval = None + try: + retval = function(self, data, False) + if retval is False: + del self._idlefuns[function] + elif retval is True: + nextsleep = 0 + elif isinstance(retval, float): + if (retval < nextsleep): + nextsleep = retval + else: + fds = fds + retval + except SystemExit: + raise + except: + import traceback + traceback.print_exc() + if retval == None: + # the function execute failed; delete it + del self._idlefuns[function] + pass + + socktimeout = self.socket.gettimeout() or nextsleep + socktimeout = min(socktimeout, nextsleep) + # Mirror what BaseServer handle_request would do + try: + fd_sets = select.select(fds, [], [], socktimeout) + if fd_sets[0] and self in fd_sets[0]: + self._handle_request_noblock() + except IOError: + # we ignore interrupted calls + pass + + # Tell idle functions we're exiting + for function, data in self._idlefuns.items(): + try: + retval = function(self, data, True) + except: + pass + self.server_close() + return + + def set_connection_token(self, token): + self.connection_token = token + +class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection): + def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = None): + self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port) + self.clientinfo = clientinfo + self.serverImpl = serverImpl + self.observer_only = observer_only + if featureset: + self.featureset = featureset + else: + self.featureset = [] + + def connect(self, token = None): + if token is None: + if self.observer_only: + token = "observer" + else: + token = self.connection.addClient() + + if token is None: + return None + + self.transport.set_connection_token(token) + return self + + def setupEventQueue(self): + self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo) + for event in bb.event.ui_queue: + self.events.queue_event(event) + + _, error = self.connection.runCommand(["setFeatures", self.featureset]) + if error: + # disconnect the client, we can't make the setFeature work + self.connection.removeClient() + # no need to log it here, the error shall be sent to the client + raise BaseException(error) + + def removeClient(self): + if not self.observer_only: + self.connection.removeClient() + + def terminate(self): + # Don't wait for server indefinitely + import socket + socket.setdefaulttimeout(2) + try: + self.events.system_quit() + except: + pass + try: + self.connection.removeClient() + except: + pass + +class BitBakeServer(BitBakeBaseServer): + def initServer(self, interface = ("localhost", 0), single_use = False): + self.interface = interface + self.serverImpl = XMLRPCServer(interface, single_use) + + def detach(self): + daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log") + del self.cooker + + def establishConnection(self, featureset): + self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset) + return self.connection.connect() + + def set_connection_token(self, token): + self.connection.transport.set_connection_token(token) + +class BitBakeXMLRPCClient(BitBakeBaseServer): + + def __init__(self, observer_only = False, token = None): + self.token = token + + self.observer_only = observer_only + # if we need extra caches, just tell the server to load them all + pass + + def saveConnectionDetails(self, remote): + self.remote = remote + + def establishConnection(self, featureset): + # The format of "remote" must be "server:port" + try: + [host, port] = self.remote.split(":") + port = int(port) + except Exception as e: + bb.warn("Failed to read remote definition (%s)" % str(e)) + raise e + + # We need our IP for the server connection. We get the IP + # by trying to connect with the server + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect((host, port)) + ip = s.getsockname()[0] + s.close() + except Exception as e: + bb.warn("Could not create socket for %s:%s (%s)" % (host, port, str(e))) + raise e + try: + self.serverImpl = XMLRPCProxyServer(host, port) + self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset) + return self.connection.connect(self.token) + except Exception as e: + bb.warn("Could not connect to server at %s:%s (%s)" % (host, port, str(e))) + raise e + + def endSession(self): + self.connection.removeClient() |
