HashTable.py

__author__ = 'MatrixRev'
import codecs
import json
import re
from collections import Mapping
from collections import defaultdict
from collections import Counter
import Levenshtein
import nltk
from nltk.corpus import stopwords
import fileinput
import string
import os

import collections
import rdflib
from rdflib.namespace import OWL, RDF, RDFS,DC,FOAF
from rdflib import Graph, Literal, Namespace, URIRef,BNode
from rdflib.plugins.memory import IOMemory
from rdflib.graph import Graph, ConjunctiveGraph
from rdflib import plugin
from tempfile import mkdtemp
from rdflib.store import Store, NO_STORE, VALID_STORE


filename='C:\\Users\\MatrixRev\\Desktop\\books\\output\\subjects.txt' # the input file
output='C:\\Users\\MatrixRev\\Desktop\\books\\output\\dictionnary.json'
filePath='C:\\Users\\MatrixRev\\Desktop\\books\\output\\foaf.rdf'
termfile='C:\\Users\\MatrixRev\\Desktop\\books\\output\\allTerms.txt'
out2='C:\\Users\\MatrixRev\\Desktop\\books\\output\\newprint.txt'
class HashTable():

    def __init__(self):
        self.allTerms={}

        self.symbols={ '**a':'Personal name','**sub':'subString',
                       '**period':'period',
                         '**location':'location','**date':'date',
                         '**b':'Numeration',
                         '**c':'Titles and other words associated with a name',
                         '**e':'Relator term',
                         '**f':'Date of a work',
                         '**g':'Miscellaneous information',
                         '**h':'Medium A media qualifier.',
                         '**j':'Attribution qualifier',
                         #Attribution information for names when the responsibility is unknown, uncertain, fictitious, or pseudonymous.
                         '**k':'Form subheading',
                         #'**l':'Language of a work (NR)',
                         '**m':'Medium of performance for music',
                         '**n':'Number of part/section of a work',
                         '**o':'Arranged statement for music',
                         #'**p':'Name of part/section of a work (R)',
                         '**q':'Fuller form of name',
                         '**r':'Key for music',
                         #'**s':'Version (NR)',
                         '**t':'Title of a work',
                         '**u':'Affiliation',
                         '**v':'Form subdivision',
                         '**x':'General subdivision',
                         '**y':'Chronological subdivision',
                         '**z':'Geographic subdivision',
                         '**0':'Authority record control number or standard number',
                        #See description of this subfield in Appendix A: Control Subfields.
                         '**2':'Source of heading or term',
                        #Code from: Subject Heading and Term Source Codes.
                         '**3':'Materials specified',
                         '**4':'Relator code',
                         '**6':'Linkage',
                        #See description of this subfield in Appendix A: Control Subfields.
                         '**8':'Field link and sequence number',
                         '=':'isA'}

        sym_keys=self.symbols.keys()
        value=self.symbols.values()



    def Relation(self):
            return ({"@bookTitle":[],
                     "relation":[],
                     "objects":[],
                     "subClass":[],
                     "semilarTerms":[],
                     "@link":[],
                     "listOFWords":[]})


    def fill_Terms(self,sub):
        terms=self.Relation()
        allterm=self.allTerms
        subStr=""
        if sub.find("***link"):
            terms['@link'].append((self.setLink(sub).strip(" ")))
        newIndex=sub.index("***link")
        sub=sub[:newIndex]
        if sub.rfind("***title"):
            terms['@bookTitle'].append('@'+self.setTitle(sub).strip(" "))
        newIndex=sub.index("***title")
        subStr=sub[:newIndex]
        #print("all sub str:",subStr)
        if self.find_relation(subStr)==True:
            for k,v in self.termValue(subStr).items():
                 terms.setdefault("relation",[]).append((k,v))

            minStr=self.CompTermKey(subStr)
            #print("the sub string is:",minStr)
            if minStr.find('(')!=-1 and minStr.find(')')!=-1:
                print("yest find para",self.if_FindParanthesis(minStr))
                terms["relation"].append(self.getParanthises(minStr))
                insidePara=self.inside_Paranthises(minStr)
                #print("insidePrara",insidePara)
                if self.if_compoTerm(insidePara)==True:
                   terms['listOFWords'].append(self.splitTerms(insidePara))
                if self.check_If_Comma(insidePara)==True:
                    listOfobjects= self.findComma(insidePara)
                    i=len(listOfobjects)-1
                    print("i",i)
                    cls=listOfobjects[i]
                    terms["objects"].append(cls)
                    for c in range(0,i):
                     subClass=listOfobjects[c]
                     terms["subClass"].append(subClass)
                    for k in range(0,len(listOfobjects)):
                        minStr=listOfobjects[k]
                        minStr=minStr.strip(" ")
                        if self.check_terms(minStr)==True:
                            allterm[minStr]=self.updateTerm(minStr,terms)
                        elif self.check_terms(minStr)==False:
                            allterm[minStr]=terms
                else:
                    minStr=insidePara
                    minStr=minStr.strip(" ")
                    if self.check_terms(minStr)==True:
                        allterm[minStr]=self.updateTerm(minStr,terms)
                    elif self.check_terms(minStr)==True:
                        allterm[minStr]=terms
            elif self.check_If_Comma(minStr)==True:
                listOfobjects= self.findComma(minStr)
                i= len(listOfobjects)-1
                print("listofbjects is :",listOfobjects[i])
                cls=listOfobjects[i]
                terms['objects'].append(cls)
                for c in range(0,i):
                    subClass=listOfobjects[c]
                    terms["subClass"].append(subClass)
                for t in range(0,len(listOfobjects)):
                    minStr=listOfobjects[t]
                    minStr=minStr.strip(" ")
                    if self.check_terms(minStr)==True:
                     #allterm[minStr].update(terms)
                      allterm[minStr]=self.updateTerm(minStr,terms)
                    elif self.check_terms(minStr)==False:
                      allterm[minStr]=terms
            else:
                if self.if_compoTerm(minStr)==True:
                   terms['listOFWords'].append(self.splitTerms(minStr))
                minStr=minStr.strip(" ")
                #minStr=minStr.lower()
                if self.check_terms(minStr)==True:
                   allterm[minStr]=self.updateTerm(minStr,terms)
                elif self.check_terms(minStr)==False:
                    allterm[minStr]=terms

        elif self.find_relation(subStr)==False:
            #subStr=subStr.lower()

            if subStr.find('(')!=-1 and subStr.find(')')!=-1:
                terms["relation"].append(self.getParanthises(subStr))
                insidePara=self.inside_Paranthises(subStr)
                #print("insidePrara",insidePara)
                if self.if_compoTerm(insidePara)==True:
                    terms['listOFWords'].append(self.splitTerms(insidePara))
                if self.check_If_Comma(insidePara)==True:
                    listOfobjects= self.findComma(insidePara)
                    i= len(listOfobjects)-1
                    cls=listOfobjects[i]
                    terms["objects"].append(cls)
                    for c in range(0,i):
                        subClass=listOfobjects[c]
                        terms["subClass"].append(subClass)
                    for t in range(0,len(listOfobjects)):
                        mterm=listOfobjects[t]

                        if self.check_terms(mterm)==True:
                            allterm[mterm]=self.updateTerm(mterm,terms)
                        elif self.check_terms(mterm)==False:
                            allterm[mterm]=terms
                else:
                    insidePara=insidePara.strip('')
                    #insidePara=insidePara.lower()
                    if self.check_terms(insidePara)==True:
                        allterm[insidePara]=self.updateTerm(insidePara,terms)
                        #print("after update",terms)
                    elif self.check_terms(insidePara)==False:
                        allterm[insidePara]=terms
                    if self.if_compoTerm(insidePara)==True:
                        terms['listOFWords'].append(self.splitTerms(insidePara))
            elif self.check_If_Comma(subStr)==True:
                #subStr=subStr.strip(" ")
                #subStr=subStr.lower()
                listOfobjects= self.findComma(subStr)
                i= len(listOfobjects)-1
                cls=listOfobjects[i]
                terms["objects"].append(cls)
                for c in range(0,i):
                    subClass=listOfobjects[c]
                    terms["subClass"].append(subClass)
                for e in range(0,len(listOfobjects)):
                    tterm=listOfobjects[e]
                    #tterm=tterm.strip(" ")
                 #   tterm=tterm.lower()
                    if self.check_terms(tterm)==True:
                        allterm[tterm]=self.updateTerm(tterm,terms)
                    elif self.check_terms(tterm)==False:
                        allterm[tterm]=terms
            else:
                #subStr=subStr.strip(" ")
                #subStr=subStr.lower()
                if self.if_compoTerm(subStr)==True:
                   terms['listOFWords'].append(self.splitTerms(subStr))
                if self.check_terms(subStr)==True:
                   allterm[subStr]=self.updateTerm(subStr,terms)
                elif self.check_terms(subStr)==False:
                    allterm[subStr]=terms
        else:
            #subStr=subStr.strip('')
            #subStr=subStr.lower()
            if self.if_compoTerm(subStr)==True:
                terms['listOFWords'].append(self.splitTerms(subStr))
            if self.check_terms(subStr)==True:
              allterm[subStr]=self.updateTerm(subStr,terms)
            elif self.check_terms(subStr)==False:
              allterm[subStr]=terms

        return allterm

    def updateTerm(self,term,rvalue):
        tvalue=self.allTerms[term]

        for key in tvalue.keys():
            for k in rvalue.keys():
                if key == k:
                    if key is 'listOFWords':
                        break
                    else:
                        for i in rvalue[k]:
                            if  i in tvalue[key]:
                                break
                            else:
                                tvalue[key].append(i)


        return tvalue

    def similarTerms(self,target):
        the_same=[]
        counter=0
        with codecs.open(termfile,'rb',encoding='utf-8')as tf:
            list_of_t=tf.readlines()
            for item in list_of_t:
              item=item.strip('\n')
              if  item!=target:
                if self.if_compoTerm(target):

                        List_target=self.splitTerms(target)
                        for t in List_target:

                            if item.find(t)!=-1:
                                 if item not in the_same:
                                     dist=Levenshtein.distance(item,target)
                                     print("the dist:",dist)
                                     if item!=target:
                                        the_same.append(item)

                            if Levenshtein.ratio(t,item)==0.8:
                                if item not in the_same:
                                    if re.fullmatch(item,target):
                                        the_same.append(item)

                #print("the ratio is ",the_ratio)

        #print("is",the_same)
        return the_same


    def setLink(self,subject):
        if subject.find("***link"):
            linkIndex=subject.index("***link")
            newIndex=linkIndex+len("***link")+1
            link=subject[newIndex:]
        return link

    def setTitle(self,sub):
        if sub.find("***title"):
            titleIndex=sub.index("***title")
            newIndex=len("***title")+titleIndex
            title=sub[newIndex:].strip(":")

        return title


    def find_relation(self, subjects):
        #print("relation is check:",subjects)
        counter=0
        is_relation=True
        sub_len = len(subjects)
        for key in self.symbols.keys():
            if subjects.rfind(key)!=-1:
                counter=counter+1

        if counter>0:
            print(counter)
            is_relation=True
        else:
            is_relation=False
        print("if the relation",is_relation)
        return is_relation


    def termValue(self,subject):
        #longVal={}
        if subject.find('etc')!=-1or subject.rfind(',etc.')!=-1 or subject.rfind(',etc')!=-1:
            subject=subject.replace('etc.',' ')
            subject=subject.replace(',etc.',' ')
            subject=subject.strip('etc')
        if subject.find('*')!=-1:
            sindex=subject.index('*')
            nsubject=subject[sindex:]
            list_of_values=list(nsubject.split('*'))
            longVal={}
            for i in list_of_values:
                if i=='':
                 list_of_values.remove(i)
            for key in self.symbols.keys():

                for sym in list_of_values:
                    sym='**'+sym
                    if sym.find(key)!=-1:
                        sym_index=sym.index(key)
                        tofsym=len(key)+sym_index
                        nsym=sym[tofsym:].strip(':').strip(' ')
                        longVal.setdefault(self.symbols[key],[]).append(nsym)




        return longVal

    def findComma(self,sub):
        newSub=[]
        if sub.find(',')!=-1:
            for s in sub.split(','):
                newSub.append(s)

        return newSub


    def getParanthises(self,sub):

       # regex = re.compile('('+')')
        longVal=collections.defaultdict(list)

        if sub.find('(')!=-1 and sub.find(')')!=-1:
            termValue=sub.strip('-')
            termValue=termValue.strip(' ')
           # print("the last char ",termValue[len(termValue)-1])
            #print("term length",len(termValue))
            #print(termValue)
            tindex=termValue.index('(')

            print("the (",tindex)
            eindex=termValue.index(')')
            print("the )",eindex)
            if eindex>tindex and eindex<len(termValue)-1:
                nValue=termValue[eindex+1:]

                longVal.setdefault('aspect_of',[]).append(nValue)
            elif tindex>eindex and tindex<len(termValue)-1:
                nValue=termValue[tindex+1:]
                #longVal.append({"aspect_of":nValue})
                longVal.setdefault('aspect_of',[]).append(nValue)
            elif eindex==len(termValue)-1and tindex<eindex:
                nValue=termValue[:tindex-1]
                #longVal.append({"aspect_of":nValue})
                longVal.setdefault('aspect_of',[]).append(nValue)
            elif tindex==len(termValue)-1and tindex>eindex:
                nValue=termValue[:eindex-1]
               # longVal.append({"aspect_of ":nValue})
                longVal.setdefault('aspect_of',[]).append(nValue)
        return longVal

    def inside_Paranthises(self,sub):
        term=""
        if sub.rfind('(')!=-1 and sub.rfind(')')!=-1:
            s=sub.index('(')
            f=sub.index(')')
            print("start",s,"end with",f)
            if s>f:

                term=sub[f:s]
                term=term.strip(")/(")
            elif s<f:

                term=sub[s:f]
                term=term.strip(")/(")
        return term
    '''
    def CompTermKey(self,sub):
        NumOfKeys=0
        term=""
        counter=0
        #regex = re.compile('('+')')
        kindex=[]
        if sub.find('***Title')!=-1:
         title_index=sub.index('***Title')
         sub=sub[:title_index]

        for key in self.symbols.keys():
            if sub.find(key)!=-1:
                key_length=len(key)
                key_index=sub.index(key)
                kindex.append(key_index)
                counter=counter+1
                print('the key',key_index)

        if len(kindex)>0:
            for i in range(len(kindex)):
              min=kindex[0]
            if kindex[i]<min:
              min=kindex[i]
            if min >0:
             term=sub[:min]


        return term
    '''
    def CompTermKey(self,sub):
        t=''
        tindex=0
        if sub.find('*')!=-1:
            tindex=sub.index('*')
            t=sub[:tindex]
        return  t
    def if_FindParanthesis(self,sub):
        #regex = re.compile('('+')')
        yes=True
        if sub.find('(')and sub.find(')'):
            yes= True
        else:
            yes=False
        return yes
    def check_If_Comma(self,subject):
        if subject.find(",")!=-1:
            return True
        else:
            return False

    def check_terms(self,term):
        existTerm= True
        if term in self.allTerms.keys():
            existTerm=True
        else:
            existTerm=False
        return existTerm
    def check_value(self,val):
        if val in self.terms.values():
            return False
        else:
            return True
    def if_compoTerm(self,sub):
        arr=sub.split()
        counter=0
        for a in arr:
            counter=counter+1
        if counter>1:
            return True
        else:
            return False
    def splitTerms(self,subString):
        allWords=[]
        for world in subString.split(' '):
            allWords.append(world)
        return (allWords)
    def printTerms(self,output):
        with codecs.open(output,'w',encoding='utf-8')as outfile:
            for item in self.allTerms.items():
                #print("term:",k,outfile)
                json.dump(item,outfile,ensure_ascii=False,indent=4)

       # return self.terms
    #def Find_similarity(self,source,target):


    def num_there(self,txt):
        counter=0
        for c in txt:
            if c.isdigit():
                counter=counter+1
        if counter>=1:
            return True
        else:
            return False
    def create_ontology(self,tr,predicate,subClass,address,booktitle):
        LDT= Namespace("http://www.JceFinalProjectOntology.com/")
        ut=Namespace("http://www.JceFinalProjectOntology.com/subject/#")
        usubClass=URIRef("http://www.JceFinalProjectOntology.com/subject/"+subClass.strip()+'#')
        #LDT.subClass=LDT[subClass]
        print(ut)
        print(usubClass)

        store=IOMemory()

        sty=LDT[predicate]
        g = rdflib.Graph(store=store,identifier=LDT)
        t = ConjunctiveGraph(store=store,identifier=ut)
        print ('Triples in graph before add: ', len(t))
        #g.add((LDT,RDF.type,RDFS.Class))
        g.add((URIRef(LDT),RDF.type,RDFS.Class))
        g.add((URIRef(LDT),RDFS.label,Literal("JFPO")))
        g.add((URIRef(LDT),RDFS.comment,Literal('class of all properties')))
        for  v in self.symbols.values():
            if self.if_compoTerm(v)==True:
                vs=self.splitTerms(v)[0]
            else:
                vs =v
            g.add((LDT[vs],RDF.type,RDF.Property))
            g.add((LDT[vs],RDFS.label,Literal('has'+vs)))
            g.add((LDT[vs],RDFS.comment,Literal(v)))
            g.add((LDT[vs],RDFS.range,OWL.Class))
            g.add((LDT[vs],RDFS.domain,Literal(vs)))
        g.bind('JFPO',LDT)
        #g.commit()
        g.serialize('trtst.rdf',format='turtle')

        t.add( (ut[tr], RDF.type,OWL.Class) )
        t.add((ut[tr],RDFS.subClassOf,OWL.Thing))
        t.add((ut[tr],RDFS.label,Literal(tr)))
        t.add((ut[tr],DC.title,Literal(booktitle)))
        t.add((ut[tr],DC.source,Literal(address)))

        t.add((ut[tr],DC[predicate],URIRef(usubClass)))
        t.add((ut[tr],LDT[predicate],RDF.Property))

        t.add((ut[tr],DC[predicate],URIRef(usubClass)))

        t.add((ut[tr],DC[predicate],URIRef(usubClass)))
        relation='has'+predicate
        t.add((ut[tr],LDT.term(predicate),URIRef(usubClass)))

        t.add( (usubClass,RDF.type,OWL.Class))
        t.add((usubClass,RDFS.subClassOf,OWL.Thing))
        t.add((usubClass,RDFS.subClassOf,URIRef(sty)))
        t.add((usubClass,RDFS.label,Literal(subClass)))

        #tc=Graph(store=store,identifier=usubClass)
        t.bind("dc", "http://http://purl.org/dc/elements/1.1/")
        t.bind('JFPO',LDT)
        t.commit()
                #print(t.serialize(format='pretty-xml'))

        t.serialize('test2.owl',format='turtle')

'''
ald=Levenshtein.editops("science computer","computer science")
print(ald)
edit=Levenshtein.distance("computer science","computer science")
dist=Levenshtein.apply_edit(ald,"science computer","computer science")
print(dist)
print(edit)
'''
table=HashTable()

rdf=table.create_ontology('naturalTresure','location','amman','bokAddr','/t/welcm.json') # function to create ontology
print(rdf)

'''
with codecs.open(output,'w',encoding='utf-8')as outfile:
        for k,v in table.allTerms.items():
            #if len(k) !=0:
                    #melon["item"]=list(k)
                #melon["Relation"]=list(v)
                json.dump({"item":k,"subClasses":v},outfile,ensure_ascii=False,indent=4)
                #json.dump(melon,outfile,ensure_ascii=False,indent=4)
''''''
with codecs.open(os.path.join(filePath,"allTerms.txt"),'w',encoding='utf-8')as termFile:
    list_of_terms= table.allTerms.keys()
    for k in list_of_terms:
        print(k,'\n',file=termFile)
with codecs.open(os.path.join(filePath,"value.json"),'w',encoding='utf-8')as valueFile:
    list_of_values=table.allTerms.values()
    for v in list_of_values:
        json.dump(v,valueFile,ensure_ascii=False,indent=4)
'''