import sys, traceback
import ZODB
from Persistence import Persistent
from ZODB.FileStorage import FileStorage, FileStorageError
from Products.ZCatalog.Catalog import Catalog
from Products.TextIndexNG.TextIndexNG import TextIndexNG
import os, time
from urlparse import urlparse

from Products.TextIndexNG.ConverterRegistry import ConverterRegistry

sys.path.append('/home/sciasbat/projects/fsindex/modules')
from UrlObject import UrlObjectBase

class FileObject(UrlObjectBase, Persistent):

    meta_type='Indexable File Object'
    
    def __init__(self, path, **kw):
        """ """
        FileObject.inheritedAttribute('__init__')(self, "file://"+path, **kw)
        self._preview=''
    
    def preview(self): return self._preview
    
    def text(self):
        try:
            data=self._read_data()  #this function must be called before 
                                    #getRegisteredObject
                
            converter = ConverterRegistry.getRegisteredObject(self.info('content-type'))
            if converter and self.info('content-encoding') in ['application/x-gzip', 'gzip']:
                data=ConverterRegistry.getRegisteredObject('gzip')(data)
            data=converter(data)
        except:
            data=''            
        self._preview=data[:250]
        
        return data

    def path(self):
    
        return urlparse(self.url())[3]

class TrivialCatalog(Catalog):
    
    def __init__(self):
        Catalog.__init__(self)
        self._objects = {}

    def catalogObject(self, obj, uid):
        Catalog.catalogObject(self, obj, uid)
        _objects=self._objects
        _objects[uid] = obj
        self._objects=_objects
    
    def uncatalogObject(self, uid):
        Catalog.uncatalogObject(self,uid)
        _objects=self._objects
        del _objects[uid]
        self._objects=_objects
    
    def recatalogObject(self, uid):
         Catalog.uncatalogObject(self,uid)
         Catalog.catalogObject(self, obj, uid)
    
    def has_key(self, uid):
        return self._objects.has_key(uid)
    
    def has_file(self, file):
        return self.has_key("file://"+file)
        
    def get(self, uid, default=None):
        if uid.find("file://")!=0: uid="file://"+uid
        return self._objects.get(uid, default)
    
    def keys(self):
        return self._objects.keys()
    
    def files(self):
        def _path(x): urlparse(x)[2]
        return map(_path, self.keys())
        
    def unrestrictedTraverse(self, path):
        return self._objects[path]

class Application:
    def __init__( self, file='db.fs', extra=None, verbose=None, timed=None):
        self.file= file
        self.db  = ZODB.DB( FileStorage( file ) )
        self.co  = self.db.open()
        self.root= self.co.root()
        if self.root.has_key( 'cat' ):
            self.cat= self.root['cat']
        else:
            self.cat = TrivialCatalog()
            self.cat.aq_parent= self.root
            TI = TextIndexNG('text', extra,  caller=self.cat, verbose=verbose, timed_statistics=timed)
            if verbose: TI.debugOn()
            else: TI.debugOff()
            TI.timed_statistics = timed
            TI.catalog = self.cat
            self.cat.addIndex('text',TI)
            self.cat.addColumn('preview')
            self.cat.addColumn('path')
            self.root['cat']=self.cat
            
        get_transaction().commit()


    def index_tree(self, dirname):
        
        def dirfiles(dirname):
            return map(lambda x,y=dirname: os.path.join(y,x), os.listdir(dirname))
            
        files = dirfiles(dirname)
        
        TI=self.cat.indexes['text']
        
        ts = time.time()
        bytes = 0
        print '-'*78
        nfiles=len(files)
        
        old_files={}
        for f in self.cat.keys(): old_files[f]=0
        
        #import pdb; pdb.set_trace()
        while files:
            f = files.pop(0)
            if os.path.isdir(f):
                newfiles=dirfiles(f)
                nfiles+=len(newfiles)
                files.extend(newfiles)
            
            if not os.path.isfile(f): continue
            
            try:
                T=self.cat.get(f)
                if T is None:
                    print f,"not in the index"
                    T=FileObject(f)
                    self.cat.catalogObject(T,T.id())
                    bytes+=os.stat(f)[6]
                elif T.is_modified():
                    print f,"modified!"
                    del old_files[T.id()] #this file has been indexed
                    self.cat.recatalogObject(T.id())
                    bytes+=os.stat(f)[6]
                else:
                    print "%s is not modified"%(T.id())
                    del old_files[T.id()]

            except ImportError:
                traceback.print_exc()
                pass
    
        print "Removed files", old_files
        
        get_transaction().commit()
    
        diff = time.time() - ts
        print "%d files, total size: %d" % (nfiles, bytes)
        print "Indexing time: %5.3lf" % diff
        print 'Indexingspeed: %5.3lf KB/sec' % (1.0*bytes/diff/1024.0)
        
        try: self.db.pack()
        except FileStorageError:
            print "Ooops, cannot pack!"
            pass
        
        for x in TI.allSettingOptions():
            print "%25s = %s" % (x,TI.getSetting(x))



class Extra:
    pass


extra=Extra()
extra.useConverters=0
a=Application(extra=extra, verbose=0, timed=0)
a.index_tree('/home/sciasbat/docs/lsi/')