diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index b419f0a3..2714441c 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -30,7 +30,7 @@ jobs: run: | python -m pip install --upgrade pip pip install flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements.txt ]; then pip install --upgrade -r requirements.txt; fi - name: Lint with flake8 continue-on-error: true run: | diff --git a/requirements.txt b/requirements.txt index b85dc822..d947198a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ pandasql pysolr base36 vfb_connect -better_profanity \ No newline at end of file +better_profanity +SQLAlchemy>=1.0,<2.0 diff --git a/src/uk/ac/ebi/vfb/neo4j/KB_tools.py b/src/uk/ac/ebi/vfb/neo4j/KB_tools.py index 49628685..a9108908 100644 --- a/src/uk/ac/ebi/vfb/neo4j/KB_tools.py +++ b/src/uk/ac/ebi/vfb/neo4j/KB_tools.py @@ -6,7 +6,8 @@ import warnings import re import json -#import psycopg2 +# import psycopg2 +import pandas as pd import requests from .neo4j_tools import neo4j_connect, results_2_dict_list from .SQL_tools import get_fb_conn, dict_cursor @@ -42,6 +43,7 @@ def get_sf(iri): """Get a short form from an iri.""" return re.split('[#/]', iri)[-1] + def gen_id(idp, ID, length, id_name, use_base36=False): """ Generates an ID of form _ @@ -49,9 +51,10 @@ def gen_id(idp, ID, length, id_name, use_base36=False): ARG 2 starting ID number (int), ARG3, length of numeric portion ID, ARG4 an id:name hash""" + def gen_key(ID, length): # This function is limited to the scope of the gen_id function. dl = len(str(ID)) # coerce int to string. - k = idp+'_'+(length - dl)*'0'+str(ID) + k = idp + '_' + (length - dl) * '0' + str(ID) return k k = gen_key(ID, length) @@ -85,7 +88,6 @@ def contains_profanity(value, use_base36): return False - class kb_writer (object): def __init__(self, endpoint, usr, pwd, hard_fail=False): @@ -100,22 +102,22 @@ def _commit(self, verbose=False, chunk_length=5000): Returns REST API output. Optionally set verbosity and chunk length for commits.""" self.output = self.nc.commit_list_in_chunks( - statements=self.statements, - verbose=verbose, - chunk_length=chunk_length) + statements=self.statements, + verbose=verbose, + chunk_length=chunk_length) self.statements = [] return self.output def commit(self, verbose=False, chunk_length=5000): return self._commit(verbose, chunk_length) - def escape_string(self, strng:str): + def escape_string(self, strng: str): # backslashes need special escaping to be treated literally return re.sub(r'\\', r'\\\\', strng) - + def _add_textual_attribute(self, var, key, value): - return 'SET %s.%s = "%s" ' % (var, key, self.escape_string(value)) # Note arrangement single and double quotes - + return 'SET %s.%s = "%s" ' % (var, key, self.escape_string(value)) # Note arrangement single and double quotes + def _set_attributes_from_dict(self, var, attribute_dict): """Generates CYPHER `SET` sub-clauses from key value pairs in a dict (attribute_dict). @@ -124,26 +126,28 @@ def _set_attributes_from_dict(self, var, attribute_dict): """ # Note - may be able to simplify this by converting to a map and passing that. out = '' - for k,v in attribute_dict.items(): + for k, v in attribute_dict.items(): if type(v) == int: - out += "SET %s.%s = %d " % (var,k,v) - elif type(v) == float: - out += "SET %s.%s = %f " % (var,k,v) + out += "SET %s.%s = %d " % (var, k, v) + elif type(v) == float: + out += "SET %s.%s = %f " % (var, k, v) elif type(v) == str: - out += 'SET %s.%s = "%s" ' % (var, k, self.escape_string(v)) - elif type(v) == list: - out += 'SET %s.%s = %s ' % (var,k, str([self.escape_string(i) for i in v])) + out += 'SET %s.%s = "%s" ' % (var, k, self.escape_string(v)) + elif type(v) == list: + out += 'SET %s.%s = %s ' % (var, k, str([self.escape_string(i) for i in v])) elif type(v) == bool: - out += "SET %s.%s = %s " % (var,k, str(v)) - else: - warnings.warn("Can't use a %s as an attribute value in Cypher. Key %s Value :%s" + out += "SET %s.%s = %s " % (var, k, str(v)) + else: + warnings.warn("Can't use a %s as an attribute value in Cypher. Key %s Value :%s" % (type(v), k, (str(v)))) return out + class iri_generator(kb_writer): """ A wrapper class for generating IRIs for *OWL individuals* that don't stomp on those already in the KB. """ + def __init__(self, endpoint, usr, pwd, use_base36=False, idp='VFB', @@ -156,7 +160,6 @@ def __init__(self, endpoint, usr, pwd, acc_length=acc_length, base=base) - def _configure(self, idp, acc_length, base): self.acc_length = acc_length self.idp = idp @@ -184,7 +187,7 @@ def _configure(self, idp, acc_length, base): self.lookup.add(base36.loads(acc)) return True else: - warnings.warn("No existing ids match the pattern %s_%s" % (idp, 'n'*acc_length)) + warnings.warn("No existing ids match the pattern %s_%s" % (idp, 'n' * acc_length)) return False def set_channel_config(self): @@ -213,6 +216,7 @@ def _get_new_accession(self, start): def _gen_short_form(self, accession): return self.idp + '_' + str(accession).zfill(self.acc_length) + class kb_owl_edge_writer(kb_writer): """A class wrapping methods for updating imported entities in the KB. Constructor: kb_owl_edge_writer(endpoint, usr, pwd) @@ -237,8 +241,8 @@ def check_properties(self): for k, v in self.properties.items(): # If this was only operating on Neo3, could just grab all node properties as a map. statements.append( - "OPTIONAL MATCH (r:Property { %s: '%s' }) " - "RETURN r.iri as iri, r.short_form as short_form, " + "OPTIONAL MATCH (r:Property { %s: '%s' }) " + "RETURN r.iri as iri, r.short_form as short_form, " "r.label as label, '%s' as key, '%s' as match_on" % ( v['match_on'], k, k, v['match_on']) ) @@ -269,9 +273,9 @@ def _report_missing_property(self, prop): """Remove triples using specified prop and warn.""" for t in self.triples.pop(prop): m = "Unknown property %s: Can't add triple %s, %s, %s." % (prop, - t['o'], - prop, - t['s']) + t['o'], + prop, + t['s']) self.log.append(m) warnings.warn(m) @@ -309,7 +313,7 @@ def _construct_triples(self): else: rel = rel_map[t['match_on']] - out = "OPTIONAL MATCH (s%s { %s:'%s' }) " % (t['stype'],t['match_on'], t['s']) + out = "OPTIONAL MATCH (s%s { %s:'%s' }) " % (t['stype'], t['match_on'], t['s']) out += "OPTIONAL MATCH (o%s { %s:'%s' }) " % (t['otype'], t['match_on'], t['o']) out += "FOREACH (a IN CASE WHEN s IS NOT NULL THEN [s] ELSE [] END | " \ "FOREACH (b IN CASE WHEN o IS NOT NULL THEN [o] ELSE [] END | " @@ -317,8 +321,8 @@ def _construct_triples(self): out += "MERGE (a)-[re:%s]->(b) SET re.type = '%s' " % (rel, t['rtype']) # Might need work? else: out += "MERGE (a)-[re:%s { %s: '%s' }]->(b) " % (t['rtype'], - t['match_on'], - rel) + t['match_on'], + rel) out += self._set_attributes_from_dict('re', t['edge_annotations']) # For each of label, iri, short_form; Check if available; Check if used in match @@ -326,14 +330,13 @@ def _construct_triples(self): # This is needed as cypher doesn't like property used in merge is also set in same statement. if rel_map['label'] and ((not t['match_on'] == 'label') or t['safe_label_edge']): out += "SET re.label = '%s' " % rel_map['label'] - if rel_map['short_form'] and ((not t['match_on'] == 'short_form' ) or t['safe_label_edge']): + if rel_map['short_form'] and ((not t['match_on'] == 'short_form') or t['safe_label_edge']): out += "SET re.short_form = '%s' " % rel_map['short_form'] if rel_map['iri'] and ((not t['match_on'] == 'iri') or t['safe_label_edge']): out += "SET re.iri = '%s' " % rel_map['iri'] out += ")) RETURN { `%s`: count(s), `%s`: count(o) } as match_count" % (t['s'], t['o']) self.statements.append(out) - def _add_related_edge(self, s, r, o, stype, otype, edge_annotations=None, match_on="iri", safe_label_edge=True): if edge_annotations is None: @@ -387,7 +390,6 @@ def add_xref(self, s, xref, stype=''): match_on='short_form', safe_label_edge=True) - def add_anon_type_ax(self, s, r, o, edge_annotations=None, match_on="iri", safe_label_edge=True): """Add OWL anonymous type axiom to statement stack. @@ -400,8 +402,8 @@ def add_anon_type_ax(self, s, r, o, edge_annotations=None, """ if edge_annotations is None: edge_annotations = {} self._add_related_edge(s, r, o, stype=":Individual", otype=":Class", - edge_annotations = edge_annotations, - match_on = match_on, + edge_annotations=edge_annotations, + match_on=match_on, safe_label_edge=safe_label_edge) def add_named_type_ax(self, s, o, match_on="iri", edge_annotations=None): @@ -434,9 +436,9 @@ def add_anon_subClassOf_ax(self, s, r, o, edge_annotations=None, """ if edge_annotations is None: edge_annotations = {} - self._add_related_edge(s, r, o, stype = ":Class", otype = ":Class", - edge_annotations = edge_annotations, - match_on = match_on, + self._add_related_edge(s, r, o, stype=":Class", otype=":Class", + edge_annotations=edge_annotations, + match_on=match_on, safe_label_edge=safe_label_edge) def add_named_subClassOf_ax(self, s, o, match_on="iri"): @@ -451,7 +453,7 @@ def add_named_subClassOf_ax(self, s, o, match_on="iri"): "MERGE (a)-[:SUBCLASSOF]->(b) " out += ")) RETURN { `%s`: count(s), `%s`: count(o) } as match_count" % (s, o) self.statements.append(out) - + def commit(self, verbose=False, chunk_length=5000): """Check prroperties; construct triples for all properties present; commit all edge additions (triples and duples) and test success. @@ -461,7 +463,7 @@ def commit(self, verbose=False, chunk_length=5000): self.check_properties() self._construct_triples() self._commit(verbose, chunk_length) - self.test_edge_addition() # Do something with return value? + self.test_edge_addition() # Do something with return value? # At this point - resetting all attributes except connection to default. # Better practice to just make a new object? out = self.output @@ -486,24 +488,25 @@ def test_edge_addition(self): else: return True + class node_importer(kb_writer): """A class wrapping methods for updating imported entities in the KB, e.g. from ontologies, FlyBase, CATMAID. Constructor: owl_import_updater(endpoint, usr, pwd) """ - + def add_constraints(self, uniqs=None, indexes=None): """Specify addition uniqs and indexes via dicts. { label : [attributes] } """ if uniqs is None: uniqs = {} if indexes is None: indexes = {} - for k,v in uniqs.items(): + for k, v in uniqs.items(): for a in v: - self.statements.append("CREATE CONSTRAINT ON (n:%s) ASSERT n.%s IS UNIQUE" % (k,a)) - for k,v in indexes.items(): + self.statements.append("CREATE CONSTRAINT ON (n:%s) ASSERT n.%s IS UNIQUE" % (k, a)) + for k, v in indexes.items(): for a in v: - self.statements.append("CREATE INDEX ON :%s(%s)" % (k,a)) - + self.statements.append("CREATE INDEX ON :%s(%s)" % (k, a)) + def add_default_constraint_set(self, labels): """SETS iri and short_form as uniq, indexes label""" uniqs = {} @@ -513,7 +516,7 @@ def add_default_constraint_set(self, labels): indexes[l] = ['label'] self.add_constraints(uniqs, indexes) self.commit() - + def add_node(self, labels, IRI, attribute_dict=None): """Adds or updates a node. Node uniqueness specified by IRI + labels. @@ -529,12 +532,12 @@ def add_node(self, labels, IRI, attribute_dict=None): attribute_dict=attribute_dict) self.statements.append(statement) - - def update_from_obograph(self, file_path = '', url = '', include_properties=False, commit=True): + def update_from_obograph(self, file_path='', url='', include_properties=False, commit=True): """Update property and class nodes from an OBOgraph file (currently does not distinguish OPs from APs!) Only updates from primary graph (i.e. ignores imports) """ + ## Get JSON, assuming only primary graph should be used for updating ## ie: imports ignored. @@ -553,7 +556,7 @@ def obs_check(nod): else: return False - if file_path: + if file_path: f = open(file_path, 'r') obographs = json.loads(f.read()) f.close() @@ -562,10 +565,10 @@ def obs_check(nod): r = requests.get(url) if r.status_code == 200: obographs = r.json() - primary_graph = obographs['graphs'][0] # Add a check for success here! + primary_graph = obographs['graphs'][0] # Add a check for success here! else: - warnings.warn("URL connection issue %s %s for %s" % (r.status_code, - r.reason, url)) + warnings.warn("URL connection issue %s %s for %s" % (r.status_code, + r.reason, url)) return False else: warnings.warn('Please provide a file_path or a URL') @@ -583,8 +586,8 @@ def obs_check(nod): continue # Split URL -> base & short_form m = re.findall('.+(#|/)(.+?)$', node['id']) - attribute_dict['short_form'] = m[0][1] - if 'lbl' in node.keys(): attribute_dict['label']= node['lbl'] + attribute_dict['short_form'] = m[0][1] + if 'lbl' in node.keys(): attribute_dict['label'] = node['lbl'] if 'meta' in node.keys(): if obs_check(node): attribute_dict['is_obsolete'] = obs_check(node) @@ -603,7 +606,7 @@ def check_for_obsolete_nodes_in_use(self): q = results_2_dict_list(self.nc.commit_list([m])) if q: for r in q: - warnings.warn("%s, %s is obsolete but in use." % + warnings.warn("%s, %s is obsolete but in use." % (r['c.label'], r['c.iri'])) obsolete_iris = [r['c.iri'] for r in q] return list(set(obsolete_iris)) @@ -616,6 +619,7 @@ def merge_obsoletes(self, ob_term_ids, graph): Maps a list of full length IRIs to their 'term replaced by', where possible. Produces cypher commands to transfer annotations in VFB to the replacement terms. """ + def convert_to_short_form(iri): """ Convert any type of id to a short form (with an underscore). @@ -669,8 +673,8 @@ def command_writer(old_id, new_id): for n in graph['nodes']: if i in n['id']: try: - consider_list = [convert_to_short_form(p['val']) for p in n['meta']['basicPropertyValues'] if - p['pred'] == "http://www.geneontology.org/formats/oboInOwl#consider"] + consider_list = [convert_to_short_form(p['val']) for p in n['meta']['basicPropertyValues'] + if p['pred'] == "http://www.geneontology.org/formats/oboInOwl#consider"] failed_mapping_dict[convert_to_short_form(i)] = consider_list consider_all_shortids.extend(consider_list) except KeyError: @@ -751,6 +755,46 @@ def migrate_features_to_new_ids(self, d): """STUB""" return + def update_genotypes(self): + """Checks whether obsolete FB features are used in genotypes. + Updates genotypes (using add_genotype) if feature labels changed. + Should be run after update_current_features_from_FlyBase.""" + + # check for obsolete features + s = ["MATCH (h:Feature:Class)<-[r {short_form:'has_part'}]-(g)-[:INSTANCEOF]->(x {short_form:'GENO_0000536'}) " + "WHERE h.is_obsolete=true RETURN DISTINCT g.short_form, g.synonyms, g.label, h.short_form, h.label"] + r = self.nc.commit_list(s) + rd = results_2_dict_list(r) + if rd: + obs_gen = pd.DataFrame(rd) + print("Genotypes linked to obsolete features:") + print(obs_gen) + else: + print("No obsolete features linked to genotypes.") + + # check for changed labels - not possible for synonym to need updating unless associated with new feature + # associating with new feature using add_genotype would automatically update synonym + s = ["MATCH (h:Feature:Class)<-[r {short_form:'has_part'}]-(g)-[:INSTANCEOF]->(x {short_form:'GENO_0000536'}) " + "RETURN DISTINCT g.short_form, g.synonyms, g.label, " + "COLLECT(h.short_form) AS FBIDs, COLLECT(h.label) AS labels"] + r = self.nc.commit_list(s) + rd = results_2_dict_list(r) + genotype_details = pd.DataFrame(rd) + genotype_details['expected_label'] = genotype_details['labels'].apply( + lambda x: 'genotype consisting of ' + ', '.join(sorted(x))) + changed_labels = list(genotype_details[ + genotype_details['g.label'] != genotype_details['expected_label']]['g.short_form']) + if changed_labels: + print('Some genotypes have new feature labels') + # need to import FeatureMover locally to avoid circular imports + from uk.ac.ebi.vfb.neo4j.flybase2neo.feature_tools import FeatureMover + fm = FeatureMover(endpoint=self.nc.base_uri, usr=self.nc.usr, pwd=self.nc.pwd) + for g in changed_labels: + print('updating genotype ' + g) + fm.add_genotype(short_form=g) + else: + print('No genotypes with updated feature labels') + class EntityChecker(kb_writer): @@ -808,7 +852,7 @@ def roll_new_entity_check(self, labels, query, match_on='short_form'): def _check_should_not_exist(self, hard_fail=False): self.statements.extend(self.should_not_exist) self.should_not_exist.clear() - return(self._check("Already in DB: ", exists=False, hard_fail=hard_fail)) + return (self._check("Already in DB: ", exists=False, hard_fail=hard_fail)) def _check_should_exist(self, hard_fail=False): self.statements.extend(self.should_exist) @@ -823,7 +867,6 @@ def check(self, hard_fail=False): else: return True - def _check(self, error_message, exists=True, hard_fail=False): """Run checks in the stack then empty the stack. If hard_fail = True, raise exception if any check in the stack fails.""" @@ -845,12 +888,13 @@ def _check(self, error_message, exists=True, hard_fail=False): return False else: return True - + + class KB_pattern_writer(object): """A wrapper class for adding subgraphs following some pre-specified schema pattern. """ - + def __init__(self, endpoint, usr, pwd, use_base36=False): self.ew = kb_owl_edge_writer(endpoint, usr, pwd) self.ni = node_importer(endpoint, usr, pwd) @@ -871,7 +915,7 @@ def __init__(self, endpoint, usr, pwd, use_base36=False): 'is specified output of': 'http://purl.obolibrary.org/obo/OBI_0000312', 'hasDbXref': 'http://www.geneontology.org/formats/oboInOwl#hasDbXref', 'has_source': 'http://purl.org/dc/terms/source' - } + } self.class_lookup = { 'computer graphic': 'http://purl.obolibrary.org/obo/FBbi_00000224', @@ -996,7 +1040,7 @@ def add_anatomy_image_set(self, query=dataset) if dbxref_strings: # Add checking dbxref strings for ':' - dbxrefs.update({x.split(':')[0]:x.split(':')[1] for x in dbxref_strings}) + dbxrefs.update({x.split(':')[0]: x.split(':')[1] for x in dbxref_strings}) for k in dbxrefs.keys(): self.ec.roll_entity_check(labels=['Individual'], @@ -1048,11 +1092,10 @@ def add_anatomy_image_set(self, if template == 'self': self_labels.append("Template") - self.ni.add_node(labels=self_labels, IRI=anat_id['iri'], attribute_dict=anatomy_attributes) - #dataset_short_form = self.ni.nc.commit_list(["MATCH (ds:DataSet) WHERE ds.label = %s RETURN ds.short_form" % dataset]) + # dataset_short_form = self.ni.nc.commit_list(["MATCH (ds:DataSet) WHERE ds.label = %s RETURN ds.short_form" % dataset]) self.ew.add_annotation_axiom(s=anat_id[match_on], r='source', o=dataset, @@ -1083,17 +1126,16 @@ def add_anatomy_image_set(self, attribute_dict={'label': label + '_c'} ) # Add a query to look up template channel, assuming template anat ind spec - #q = "MATCH (c:Individual)-[:Related { short_form : 'depicts' }]" \ + # q = "MATCH (c:Individual)-[:Related { short_form : 'depicts' }]" \ # "->(t:Individual { iri : '%s' }) RETURN c.iri" % template - #x = results_2_dict_list(self.ni.nc.commit_list([q])) - #template = x['c.iri'] + # x = results_2_dict_list(self.ni.nc.commit_list([q])) + # template = x['c.iri'] # Add typing as channel. This takes no vars so match_on can be fixed. self.ew.add_named_type_ax(s=channel_id['short_form'], o='VFBext_0000014', match_on='short_form') - # Imaging modality - currently works on internal lookup in script. Should probably be dynamic with DB self.ew.add_anon_type_ax(s=channel_id['iri'], r=self.relation_lookup['is specified output of'], @@ -1132,7 +1174,7 @@ def add_anatomy_image_set(self, o=ax[1], match_on='short_form') - return {'channel': channel_id, 'anatomy': anat_id } + return {'channel': channel_id, 'anatomy': anat_id} def add_dataSet(self, name, license, @@ -1171,7 +1213,7 @@ def add_dataSet(self, name, if not self.ec.check(): return False - dataset_id = {'iri': map_iri('data') + short_form , 'short_form': short_form } + dataset_id = {'iri': map_iri('data') + short_form, 'short_form': short_form} self.ni.add_node(labels=['Individual', 'DataSet'], IRI=dataset_id['iri'], attribute_dict={ @@ -1179,8 +1221,8 @@ def add_dataSet(self, name, 'short_form': short_form, 'description': [description], 'dataset_spec_text': [dataset_spec_text], - 'schema': schema }) -# self.ni.commit() + 'schema': schema}) + # self.ni.commit() self.ew.add_annotation_axiom(s=short_form, r='license', o=license, @@ -1207,17 +1249,12 @@ def add_dataSet(self, name, return dataset_id - - - - - # Specs for a fb_feature_update ## Pull current feature nodes from DB # query = "SELECT uniquename, name, is_obsolete from feature" -#class fb_feature_update(kb_writer): - +# class fb_feature_update(kb_writer): + # def add_ind(self, iri, short_form, label, synonyms = [], additional_attributes = {}): # out = "MERGE (i:Individual { IRI: '%s'} ) " \ diff --git a/src/uk/ac/ebi/vfb/neo4j/flybase2neo/feature_tools.py b/src/uk/ac/ebi/vfb/neo4j/flybase2neo/feature_tools.py index b80f0674..50c7a224 100644 --- a/src/uk/ac/ebi/vfb/neo4j/flybase2neo/feature_tools.py +++ b/src/uk/ac/ebi/vfb/neo4j/flybase2neo/feature_tools.py @@ -1,5 +1,7 @@ from .fb_tools import FB2Neo, dict_list_2_dict +from ..neo4j_tools import neo4j_connect, results_2_dict_list from ...curie_tools import map_iri +from ..KB_tools import iri_generator import re import pandas as pd import warnings @@ -38,7 +40,6 @@ def map_feature_type(fbid, ftype): # Using named tuples to standardise immutable objects for holding data. - Feature = collections.namedtuple('Feature', ['symbol', 'fbid', 'synonyms', # list of synonyms @@ -69,7 +70,6 @@ class FeatureRelation: eps: Dict[str, Node] = field(default_factory=dict) - # For splits # pub FBex tg comment split # + al @@ -81,15 +81,12 @@ class FeatureRelation: # short_form IS a compound key! We know one half at the start, and whether it is DBD or AD, but we only get the other half just before the addition. Solution is to have a separate, bespoke job to update lookup table. This can happen during Split object generation. - - - - class FeatureMover(FB2Neo): def name_synonym_lookup(self, fbids): """Takes a list of fbids, returns a dictionary of Feature objects, keyed on fbid. Note - makes unicode name primary. Makes everything else a synonym.""" + # Limitation - no concept of type: name (as opposed to symbol). def proc_feature(d, ds): @@ -338,8 +335,7 @@ def add_feature_relations(self, triples, assume_subject=True, commit=True): self.ew.commit() return FeatureRelation(features=objects_pdm, edges=triples) - def generate_expression_patterns(self, features, feature_objects=False, add_features =True, commit=True): - + def generate_expression_patterns(self, features, feature_objects=False, add_features=True, commit=True): """Takes a list of features as input, generates expression patterns for these features. @@ -357,7 +353,6 @@ def generate_expression_patterns(self, features, feature_objects=False, add_feat warnings.warn("No features provided.") return False - eps = {} triples = [] @@ -381,7 +376,6 @@ def gen_ep_feat(feat): ad = {'label': ep.label, 'synonyms': ep.synonyms} eps[ep.short_form] = ep, - # Generate label = 'label . expression pattern' # Add node # Specification of labels here assumes existing nodes correctly labelled. Should be safe in p2 @@ -411,8 +405,7 @@ def gen_ep_feat(feat): # Better standardise output here? return FeatureRelation(features=features, edges=triples, eps=eps) - - def gen_split_ep_feat(self, splits, add_feats = True, add_feature_details=False, commit=True): + def gen_split_ep_feat(self, splits, add_feats=True, add_feature_details=True, commit=True): """Adds split expression pattern nodes to Neo following Returns a dict of feature objects keyed on schema: (sep)-[:has_hemidriver]->(construct). @@ -435,17 +428,15 @@ def gen_split_ep_feat(self, splits, add_feats = True, add_feature_details=False, else: feats = self.name_synonym_lookup([s.ad, s.dbd]) - short_form = 'VFBexp_' + s.dbd + s.ad iri = map_iri('vfb') + short_form ad = {'label' : feats[s.dbd].label + ' ∩ ' + feats[s.ad].label +' expression pattern', 'synonyms': list(s.synonyms), 'description': ['The sum of all cells at the intersection between ' - 'the expression patterns of %s and' - ' %s.' % (feats[s.dbd].label, - feats[s.ad].label)]} - + 'the expression patterns of %s and' + ' %s.' % (feats[s.dbd].label, + feats[s.ad].label)]} out[short_form] = {'attributes': ad, 'iri': iri, short_form: short_form, 'xrefs': s.xrefs} @@ -473,16 +464,151 @@ def gen_split_ep_feat(self, splits, add_feats = True, add_feature_details=False, o=s.dbd, match_on='short_form') - - if commit: self.ni.commit() self.ew.commit() return out + def add_genotype(self, genotype_components=None, add_feats=True, add_feature_details=True, + commit=True, verbose=False, short_form=None): + """Adds/updates genotypes. + schema: (genotype)-[:has_part]->(allele) + genotype_components should be a list of FB IDs (if specified). + Returns dictionary for genotype if genotype is added or updated {short_form: '', label: '', synonyms: ''}. + Specifying the short_form using short_form= can update or create a genotype with this short_form. + A short_form may be specified with no genotype_components to update using the existing components. + If a genotype composed of genotype_components already exists, no changes will be made to the database. If no + short form was specified, the matching genotype will be returned, otherwise False will be returned. + """ + # look for existing node if short_form given + if short_form: + q = [("MATCH (h:Feature:Class)<-[r {short_form:'has_part'}]-(g {short_form:'%s'})-[:INSTANCEOF]->" + "(x {short_form:'GENO_0000536'}) RETURN DISTINCT COLLECT(h.short_form) AS features" % short_form)] + r = self.nc.commit_list(q) + rd = results_2_dict_list(r) + existing_genotype_components = rd[0]['features'] + if not genotype_components: # no new feat + if verbose: + print("No features specified, using existing features based on genotype short_form...") + if not existing_genotype_components: # no new or old feat + print("No features associated with genotype, please specify") + return False + else: # old but no new feat + if verbose: + print("Identified features:", existing_genotype_components) + genotype_components = existing_genotype_components + elif existing_genotype_components: # new feat and old feat + if sorted(existing_genotype_components) == sorted(genotype_components): + if verbose: + print("Existing genotype matches specified genotype_components") + else: # new feat different to old + for f in existing_genotype_components: + if f not in genotype_components: + s = [("MATCH (g {short_form:'%s'})-[r {short_form:'has_part'}]->" + "(f:Feature {short_form:'%s'}) DELETE r") % (short_form, f)] + self.ew.statements.extend(s) + if verbose: + print("Feature %s will no longer be associated with this genotype" % f) + + elif not genotype_components: + print("Genotype short_form or genotype_components must be specified") + return False + # get features in genotype_components from FlyBase + if add_feats: + if add_feature_details: + feats = self.add_features(genotype_components) + else: + feats = self.name_synonym_lookup(genotype_components) + for k, v in feats.items(): + # labels is neo4j node labels + self.ni.add_node(labels=['Class', 'Feature'], + IRI=map_iri('fb') + k, + attribute_dict={'label': v.label}) + else: + feats = self.name_synonym_lookup(genotype_components) + + # synonym (FlyBase IDs) + feat_IDs = [n.short_form for n in feats.values()] # list of FB IDs of features in genotype_components + feat_IDs.sort() + genotype_synonym = 'genotype consisting of ' + ', '.join(feat_IDs) + + # check whether synonym already exists on a VFBgeno + nc = neo4j_connect(base_uri=self.ni.nc.base_uri, usr=self.ni.nc.usr, pwd=self.ni.nc.pwd) + + q = [("MATCH (n:Individual) WHERE n.short_form =~ 'VFBgeno_[0-9]{8}' AND n.synonyms = '%s' " + "RETURN n.short_form AS short_form, n.label AS label, n.synonyms AS synonyms" + % genotype_synonym)] + + if verbose: + print("Checking for this genotype in database, running query: " + q[0]) + + r = nc.commit_list(statements=q) + existing_genotype = results_2_dict_list(r) + + # handling genotype with matching synonym + if existing_genotype: + if existing_genotype[0]['short_form'] != short_form: # different genotype has same components + print("Genotype with specified features already exists:") + print(existing_genotype[0]) + print("No changes committed to database") + if short_form: # cannot make/update desired node + return False + else: # do not need to make a new node - returns existing node + return existing_genotype[0] + elif verbose and short_form: # match is to itself - continuing to update label + print("No conflict with existing genotypes, updating...") + elif verbose: + print("Genotype not already in database") + + # IRI + if short_form: + iri = {'short_form': short_form, 'iri': map_iri('vfb') + short_form} + else: + genotype_iri = iri_generator(endpoint=self.ni.nc.base_uri, usr=self.ni.nc.usr, + pwd=self.ni.nc.pwd, idp='VFBgeno') + # these are the endpoint, usr and pwd used to set up self (FeatureMover object), idp is namespace + + iri = genotype_iri.generate(start=0) + # iri['iri'] is long form, iri['short_form'] is short form + + # label + feat_names = [n.label for n in feats.values()] # list of labels of features in genotype_components + feat_names.sort() + genotype_name = 'genotype consisting of ' + ', '.join(feat_names) + + new_genotype = {'short_form': iri['short_form'], 'label': genotype_name, 'synonym': genotype_synonym} + + # add genotype node + self.ni.add_node(labels=['Individual'], + IRI=iri['iri'], + attribute_dict={'label': genotype_name, 'synonyms': genotype_synonym}) + + # make instance of 'genotype' + self.ew.add_named_type_ax(s=iri['short_form'], + o='GENO_0000536', # 'genotype' from GENO ontology + match_on='short_form') + + # add features as parts of genotype + for f in feats: + self.ew.add_annotation_axiom(s=iri['short_form'], + stype=':Individual', + otype=':Class', + r='has_part', + o=f, + match_on='short_form') - - - + if commit: + node_commit_out = self.ni.commit(verbose=verbose) + if not node_commit_out: + print("Problem loading nodes.") + return False + edge_commit_out = self.ew.commit(verbose=verbose) + if not edge_commit_out: + print("Problem loading edges.") + return False + if verbose: + print("Genotype is now:", new_genotype) + + return new_genotype