root/tags/1.3/archive.py

Revision 43, 15.1 KB (checked in by giovannibajo, 5 years ago)

Switch to GPL license (with exception)
  • Property svn:eol-style set to native
Line 
1# Copyright (C) 2005, Giovanni Bajo
2# Based on previous work under copyright (c) 2002 McMillan Enterprises, Inc.
3#
4# This program is free software; you can redistribute it and/or
5# modify it under the terms of the GNU General Public License
6# as published by the Free Software Foundation; either version 2
7# of the License, or (at your option) any later version.
8#
9# In addition to the permissions in the GNU General Public License, the
10# authors give you unlimited permission to link or embed the compiled
11# version of this file into combinations with other programs, and to
12# distribute those combinations without any restriction coming from the
13# use of this file. (The General Public License restrictions do apply in
14# other respects; for example, they cover modification of the file, and
15# distribution when not linked into a combine executable.)
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License
23# along with this program; if not, write to the Free Software
24# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
25
26# subclasses may not need marshal or struct, but since they're
27# builtin, importing is safe.
28#
29# While an Archive is really an abstraction for any "filesystem
30# within a file", it is tuned for use with imputil.FuncImporter.
31# This assumes it contains python code objects, indexed by the
32# the internal name (ie, no '.py').
33# See carchive.py for a more general archive (contains anything)
34# that can be understood by a C program.
35
36_verbose = 0
37_listdir = None
38_environ = None
39
40# **NOTE** This module is used during bootstrap. Import *ONLY* builtin modules.
41import marshal
42import struct
43import imp
44import sys
45
46_c_suffixes = filter(lambda x: x[2] == imp.C_EXTENSION, imp.get_suffixes())
47
48for nm in ('nt', 'posix', 'dos', 'os2', 'mac'):
49    if nm in sys.builtin_module_names:
50        mod = __import__(nm)
51        _listdir = mod.listdir
52        _environ = mod.environ
53        break
54
55if hasattr(sys, 'version_info'):
56    versuffix = '%d%d'%(sys.version_info[0],sys.version_info[1])
57else:
58    vers = sys.version
59    dot1 = dot2 = 0
60    for i in range(len(vers)):
61        if vers[i] == '.':
62            if dot1:
63                dot2 = i
64                break
65            else:
66                dot1 = i
67    else:
68        dot2 = len(vers)
69    versuffix = '%s%s' % (vers[:dot1], vers[dot1+1:dot2])
70
71if "-vi" in sys.argv[1:]:
72    _verbose = 1
73
74class Archive:
75    """ A base class for a repository of python code objects.
76        The extract method is used by imputil.ArchiveImporter
77        to get code objects by name (fully qualified name), so
78        an enduser "import a.b" would become
79          extract('a.__init__')
80          extract('a.b')
81    """
82    MAGIC = 'PYL\0'
83    HDRLEN = 12        # default is MAGIC followed by python's magic, int pos of toc
84    TOCPOS = 8
85    TRLLEN = 0        # default - no trailer
86    TOCTMPLT = {}     #
87    os = None
88    _bincache = None
89    def __init__(self, path=None, start=0):
90        "Initialize an Archive. If path is omitted, it will be an empty Archive."
91        self.toc = None
92        self.path = path
93        self.start = start
94        import imp
95        self.pymagic = imp.get_magic()
96        if path is not None:
97            self.lib = open(self.path, 'rb')
98            self.checkmagic()
99            self.loadtoc()
100
101            ####### Sub-methods of __init__ - override as needed #############
102    def checkmagic(self):
103        """ Overridable.
104            Check to see if the file object self.lib actually has a file
105            we understand.
106        """
107        self.lib.seek(self.start)       #default - magic is at start of file
108        if self.lib.read(len(self.MAGIC)) != self.MAGIC:
109            raise RuntimeError, "%s is not a valid %s archive file" \
110              % (self.path, self.__class__.__name__)
111        if self.lib.read(len(self.pymagic)) != self.pymagic:
112            raise RuntimeError, "%s has version mismatch to dll" % (self.path)
113        self.lib.read(4)
114
115    def loadtoc(self):
116        """ Overridable.
117            Default: After magic comes an int (4 byte native) giving the
118            position of the TOC within self.lib.
119            Default: The TOC is a marshal-able string.
120        """
121        self.lib.seek(self.start + self.TOCPOS)
122        (offset,) = struct.unpack('=i', self.lib.read(4))
123        self.lib.seek(self.start + offset)
124        self.toc = marshal.load(self.lib)
125
126        ######## This is what is called by FuncImporter #######
127        ## Since an Archive is flat, we ignore parent and modname.
128        #XXX obsolete - imputil only code
129        ##  def get_code(self, parent, modname, fqname):
130        ####    if _verbose:
131        ####      print "I: get_code(%s, %s, %s, %s)" % (self, parent, modname, fqname)
132        ##    iname = fqname
133        ##    if parent:
134        ##        iname = '%s.%s' % (parent.__dict__.get('__iname__', parent.__name__), modname)
135        ####        if _verbose:
136        ####            print "I: get_code: iname is %s" % iname
137        ##    rslt = self.extract(iname) # None if not found, (ispkg, code) otherwise
138        ####    if _verbose:
139        ####        print 'I: get_code: rslt', rslt
140        ##    if rslt is None:
141        ####      if _verbose:
142        ####          print 'I: get_code: importer', getattr(parent, "__importer__", None),'self',self
143        ##      # check the cache if there is no parent or self is the parents importer
144        ##      if parent is None or getattr(parent, "__importer__", None) is self:
145        ####            if _verbose:
146        ####                print 'I: get_code: cached 1',iname
147        ##            file, desc = Archive._bincache.get(iname, (None, None))
148        ####            if _verbose:
149        ####                print 'I: get_code: file',file,'desc',desc
150        ##            if file:
151        ##              try:
152        ##                fp = open(file, desc[1])
153        ##              except IOError:
154        ##                pass
155        ##              else:
156        ##                module = imp.load_module(fqname, fp, file, desc)
157        ##                if _verbose:
158        ##                    print "I: import %s found %s" % (fqname, file)
159        ##                return 0, module, {'__file__':file}
160        ##      if _verbose:
161        ##          print "I: import %s failed" % fqname
162        ##
163        ##      return None
164        ##
165        ##    ispkg, code = rslt
166        ##    values = {'__file__' : code.co_filename, '__iname__' : iname}
167        ##    if ispkg:
168        ##      values['__path__'] = [fqname]
169        ##    if _verbose:
170        ##        print "I: import %s found %s" % (fqname, iname)
171        ##    return ispkg, code, values
172
173        ####### Core method - Override as needed  #########
174    def extract(self, name):
175        """ Get the object corresponding to name, or None.
176            For use with imputil ArchiveImporter, object is a python code object.
177            'name' is the name as specified in an 'import name'.
178            'import a.b' will become:
179            extract('a') (return None because 'a' is not a code object)
180            extract('a.__init__') (return a code object)
181            extract('a.b') (return a code object)
182            Default implementation:
183              self.toc is a dict
184              self.toc[name] is pos
185              self.lib has the code object marshal-ed at pos
186        """
187        ispkg, pos = self.toc.get(name, (0,None))
188        if pos is None:
189            return None
190        self.lib.seek(self.start + pos)
191        return ispkg, marshal.load(self.lib)
192
193        ########################################################################
194        # Informational methods
195
196    def contents(self):
197        """Return a list of the contents
198           Default implementation assumes self.toc is a dict like object.
199           Not required by ArchiveImporter.
200        """
201        return self.toc.keys()
202
203        ########################################################################
204        # Building
205
206        ####### Top level method - shouldn't need overriding #######
207    def build(self, path, lTOC):
208        """Create an archive file of name 'path'.
209           lTOC is a 'logical TOC' - a list of (name, path, ...)
210           where name is the internal name, eg 'a'
211           and path is a file to get the object from, eg './a.pyc'.
212        """
213        self.path = path
214        self.lib = open(path, 'wb')
215        #reserve space for the header
216        if self.HDRLEN:
217            self.lib.write('\0'*self.HDRLEN)
218
219            #create an empty toc
220
221        if type(self.TOCTMPLT) == type({}):
222            self.toc = {}
223        else:       # assume callable
224            self.toc = self.TOCTMPLT()
225
226        for tocentry in lTOC:
227            self.add(tocentry)   # the guts of the archive
228
229        tocpos = self.lib.tell()
230        self.save_toc(tocpos)
231        if self.TRLLEN:
232            self.save_trailer(tocpos)
233        if self.HDRLEN:
234            self.update_headers(tocpos)
235        self.lib.close()
236
237
238        ####### manages keeping the internal TOC and the guts in sync #######
239    def add(self, entry):
240        """Override this to influence the mechanics of the Archive.
241           Assumes entry is a seq beginning with (nm, pth, ...) where
242           nm is the key by which we'll be asked for the object.
243           pth is the name of where we find the object. Overrides of
244           get_obj_from can make use of further elements in entry.
245        """
246        if self.os is None:
247            import os
248            self.os = os
249        nm = entry[0]
250        pth = entry[1]
251        pynm, ext = self.os.path.splitext(self.os.path.basename(pth))
252        ispkg = pynm == '__init__'
253        assert ext in ('.pyc', '.pyo')
254        self.toc[nm] = (ispkg, self.lib.tell())
255        f = open(entry[1], 'rb')
256        f.seek(8)       #skip magic and timestamp
257        self.lib.write(f.read())
258
259    def save_toc(self, tocpos):
260        """Default - toc is a dict
261           Gets marshaled to self.lib
262        """
263        marshal.dump(self.toc, self.lib)
264
265    def save_trailer(self, tocpos):
266        """Default - not used"""
267        pass
268
269    def update_headers(self, tocpos):
270        """Default - MAGIC + Python's magic + tocpos"""
271        self.lib.seek(self.start)
272        self.lib.write(self.MAGIC)
273        self.lib.write(self.pymagic)
274        self.lib.write(struct.pack('=i', tocpos))
275
276class DummyZlib:
277    def decompress(self, data):
278        return data
279    def compress(self, data, lvl):
280        return data
281
282import iu
283##############################################################
284#
285# ZlibArchive - an archive with compressed entries
286#
287class ZlibArchive(Archive):
288    MAGIC = 'PYZ\0'
289    TOCPOS = 8
290    HDRLEN = 16
291    TRLLEN = 0
292    TOCTMPLT = {}
293    LEVEL = 9
294
295    def __init__(self, path=None, offset=None, level=9):
296        if path is None:
297            offset = 0
298        elif offset is None:
299            for i in range(len(path)-1, -1, -1):
300                if path[i] == '?':
301                    offset = int(path[i+1:])
302                    path = path[:i]
303                    break
304            else:
305                offset = 0
306        self.LEVEL = level
307        Archive.__init__(self, path, offset)
308        # dynamic import so not imported if not needed
309        global zlib
310        if self.LEVEL:
311            try:
312                import zlib
313            except ImportError:
314                zlib = DummyZlib()
315        else:
316            zlib = DummyZlib()
317
318
319    def extract(self, name):
320        (ispkg, pos, lngth) = self.toc.get(name, (0, None, 0))
321        if pos is None:
322            return None
323        self.lib.seek(self.start + pos)
324        try:
325            co = marshal.loads(zlib.decompress(self.lib.read(lngth)))
326        except EOFError:
327            raise ImportError, "PYZ entry '%s' failed to unmarshal" % name
328        return ispkg, co
329
330    def add(self, entry):
331        if self.os is None:
332            import os
333            self.os = os
334        nm = entry[0]
335        pth = entry[1]
336        base, ext = self.os.path.splitext(self.os.path.basename(pth))
337        ispkg = base == '__init__'
338        try:
339            txt = open(pth[:-1], 'r').read()+'\n'
340        except (IOError, OSError):
341            try:
342                f = open(pth, 'rb')
343                f.seek(8)       #skip magic and timestamp
344                bytecode = f.read()
345                marshal.loads(bytecode).co_filename # to make sure it's valid
346                obj = zlib.compress(bytecode, self.LEVEL)
347            except (IOError, ValueError, EOFError, AttributeError):
348                raise ValueError("bad bytecode in %s and no source" % pth)
349        else:
350            txt = iu._string_replace(txt, '\r\n', '\n')
351            try:
352                co = compile(txt, "%s/%s" % (self.path, nm), 'exec')
353            except SyntaxError, e:
354                print "Syntax error in", pth[:-1]
355                print e.args
356                raise
357            obj = zlib.compress(marshal.dumps(co), self.LEVEL)
358        self.toc[nm] = (ispkg, self.lib.tell(), len(obj))
359        self.lib.write(obj)
360    def update_headers(self, tocpos):
361        """add level"""
362        Archive.update_headers(self, tocpos)
363        self.lib.write(struct.pack('!i', self.LEVEL))
364    def checkmagic(self):
365        Archive.checkmagic(self)
366        self.LEVEL = struct.unpack('!i', self.lib.read(4))[0]
367
368class PYZOwner(iu.Owner):
369    def __init__(self, path):
370        self.pyz = ZlibArchive(path)
371        iu.Owner.__init__(self, path)
372    def getmod(self, nm, newmod=imp.new_module):
373        rslt = self.pyz.extract(nm)
374        if rslt is None:
375            return None
376        ispkg, co = rslt
377        mod = newmod(nm)
378        try:
379            mod.__file__ = co.co_filename
380        except AttributeError:
381            raise ImportError, "PYZ entry '%s' (%s) is not a valid code object" % (nm, repr(co))
382        if ispkg:
383            if _environ.has_key('_MEIPASS2'):
384                localpath = _environ['_MEIPASS2'][:-1]
385            else:
386                localpath = iu._os_path_dirname(self.path)
387            mod.__path__ = [self.path, localpath, iu._os_path_dirname(mod.__file__)]
388            #print "PYZOwner setting %s's __path__: %s" % (nm, mod.__path__)
389            importer = iu.PathImportDirector(mod.__path__,
390                                              {self.path:PkgInPYZImporter(nm, self),
391                                               localpath:ExtInPkgImporter(localpath, nm)},
392                                              [iu.DirOwner])
393            mod.__importsub__ = importer.getmod
394        mod.__co__ = co
395        return mod
396
397class PkgInPYZImporter:
398    def __init__(self, name, owner):
399        self.name = name
400        self.owner = owner
401    def getmod(self, nm):
402        #print "PkgInPYZImporter.getmod %s -> %s" % (nm, self.name+'.'+nm)
403        return self.owner.getmod(self.name+'.'+nm)
404class ExtInPkgImporter(iu.DirOwner):
405    def __init__(self, path, prefix):
406        iu.DirOwner.__init__(self, path)
407        self.prefix = prefix
408    def getmod(self, nm):
409        return iu.DirOwner.getmod(self, self.prefix+'.'+nm)
410
411        #XXX this should also get moved out
412        ##iu._globalownertypes.insert(0, PYZOwner)
413        ##iu.ImportManager().install()
Note: See TracBrowser for help on using the browser.