Logo Search packages:      
Sourcecode: rapid-spring version File versions  Download package

def rapid::rapid::Package::download_files (   self,
  requested_files,
  progress = None 
)
Download requested_files using the streamer.cgi interface.

    Progress is reported through the progress object, which must be
    callable (with a single argument to indicate progress _increase_),
    a setMaximum (int) setter and int maximum() getter

    streamer.cgi works as follows:
    * The client does a POST to /streamer.cgi?<hex>
      Where hex = the name of the .sdp
    * The client then sends a gzipped bitarray representing the files
      it wishes to download. Bitarray is formated in the obvious way,
      an array of characters where each file in the sdp is represented
      by the (index mod 8) bit (shifted left) of the (index div 8) byte
      of the array.
    * streamer.cgi then responds with <big endian encoded int32 length>
      <data of gzipped pool file> for all files requested. Files in the
      pool are also gzipped, so there is no need to decompress unless
      you wish to verify integrity.
    * streamer.cgi also sets the Content-Length header in the reply so
      you can implement a proper progress bar.

Definition at line 451 of file rapid.py.

                                                                :
            """ Download requested_files using the streamer.cgi interface.

                Progress is reported through the progress object, which must be
                callable (with a single argument to indicate progress _increase_),
                a setMaximum (int) setter and int maximum() getter

                streamer.cgi works as follows:
                * The client does a POST to /streamer.cgi?<hex>
                  Where hex = the name of the .sdp
                * The client then sends a gzipped bitarray representing the files
                  it wishes to download. Bitarray is formated in the obvious way,
                  an array of characters where each file in the sdp is represented
                  by the (index mod 8) bit (shifted left) of the (index div 8) byte
                  of the array.
                * streamer.cgi then responds with <big endian encoded int32 length>
                  <data of gzipped pool file> for all files requested. Files in the
                  pool are also gzipped, so there is no need to decompress unless
                  you wish to verify integrity.
                * streamer.cgi also sets the Content-Length header in the reply so
                  you can implement a proper progress bar.

            """
            # Determine which files to fetch. (as bitarray and list of files)
            requested_files = set(requested_files)
            bits = bitarray(map(lambda f: f in requested_files, self.files), endian='little')
            expected_files = filter(lambda f: f in requested_files, self.files)
            if len(expected_files) == 0:
                  return

            # Can not download from offline repository...
            if not hasattr(self.repository, 'url'):
                  raise OfflineRepositoryException()

            # Build HTTP POST data.
            postdata = gzip_string(bits.tostring())

            # Perform HTTP POST request and download and process the response.
            url = '%s/streamer.cgi?%s' % (self.repository.url, self.hex)
            with closing(self.repository.downloader.post(url, postdata)) as remote:
                  if not remote.info().has_key('Content-Length'):
                        raise StreamerFormatException('Content-Length')

                  if progress:
                        progress.setMaximum( int(remote.info()['Content-Length']) )
                        progress(0)

                  for f in expected_files:
                        size = remote.read(4)
                        if size == '': raise StreamerFormatException('size')
                        size = struct.unpack('>L', size)[0]

                        data = remote.read(size)
                        if len(data) < size: raise StreamerFormatException('data')

                        # check md5 hash
                        with closing(gzip.GzipFile(mode = 'rb', fileobj = StringIO(data))) as g:
                              if md5(g.read()).digest() != f.md5:
                                    raise StreamerFormatException('md5')

                        mkdir_p( os.path.dirname(f.pool_path) )
                        atomic_write(f.pool_path, data)

                        if progress:
                              progress(4 + size)


Generated by  Doxygen 1.6.0   Back to index