From ae68fca5a90f9abff5d711075ace94377af35a4b Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 16 Jul 2021 11:02:33 -0400 Subject: [PATCH 01/43] fix authors get filtered out by person table dates --- client/src/gql/readPersonsByYearAllCenters.js | 21 +-- ingest/modules/harvester.ts | 160 +++++++++--------- ingest/modules/queryNormalizedPeople.ts | 6 +- 3 files changed, 90 insertions(+), 97 deletions(-) diff --git a/client/src/gql/readPersonsByYearAllCenters.js b/client/src/gql/readPersonsByYearAllCenters.js index 7a3ee35e..f5f18b44 100644 --- a/client/src/gql/readPersonsByYearAllCenters.js +++ b/client/src/gql/readPersonsByYearAllCenters.js @@ -9,25 +9,18 @@ export default function readPersonsByYearAllCenters (year) { persons ( distinct_on: id, where: { - _and: [ + _or: [ { persons_organizations: { - start_date: {_lt: "${startDateLT}"} + start_date: {_lt: "${startDateLT}"}, + end_date: {_gt: "${endDateGT}"} } }, { - _or: [ - { - persons_organizations: { - end_date: {_gt: "${endDateGT}"} - } - }, - { - persons_organizations: { - end_date: {_is_null: true} - } - } - ] + persons_organizations: { + start_date: {_lt: "${startDateLT}"}, + end_date: {_is_null: true} + } } ] }, diff --git a/ingest/modules/harvester.ts b/ingest/modules/harvester.ts index d782e41d..66022c3e 100644 --- a/ingest/modules/harvester.ts +++ b/ingest/modules/harvester.ts @@ -42,90 +42,90 @@ export class Harvester { // await randomWait(0, waitInterval) // check that person start date and end date has some overlap with search date range - if (dateRangesOverlapping(person.startDate, person.endDate, searchStartDate, searchEndDate)) { - // harvest for each name variance - const nameVariances = (person.nameVariances ? person.nameVariances : []) - let normedNameVariances = [{ givenName: person.givenName, familyName: person.familyName}] - if (harvestBy === HarvestOperation.QUERY_BY_AUTHOR_NAME){ - normedNameVariances = _.concat(normedNameVariances, _.map(nameVariances, (variance) => { - return { - givenName: variance.given_name, - familyName: variance.family_name - } - })) - } - await pMap (normedNameVariances, async (name) => { - const currentHarvestSets: HarvestSet[] = [] - let searchPerson = _.cloneDeep(person) - try { - searchPerson = _.set(searchPerson, 'familyName', name.familyName) - searchPerson = _.set(searchPerson, 'givenName', name.givenName) - searchPerson = _.set(searchPerson, 'givenNameInitial', name.givenName[0]) - } catch (error) { - console.log(`Error on name variance: ${JSON.stringify(name, null, 2)}`) - throw error - } - let offset = 0 - // have to alias this because nested this call below makes this undefined - let thisHarvester = this - - let sessionState = {} - await wait(thisHarvester.ds.getDataSourceConfig().requestInterval) - harvestSet = await this.fetchPublications(searchPerson, harvestBy, sessionState, offset, searchStartDate, searchEndDate) - if (harvestSet) { - currentHarvestSets.push(harvestSet) + // RPJ Commented out to have this logic happen before this method: if (dateRangesOverlapping(person.startDate, person.endDate, searchStartDate, searchEndDate)) { + // harvest for each name variance + const nameVariances = (person.nameVariances ? person.nameVariances : []) + let normedNameVariances = [{ givenName: person.givenName, familyName: person.familyName}] + if (harvestBy === HarvestOperation.QUERY_BY_AUTHOR_NAME){ + normedNameVariances = _.concat(normedNameVariances, _.map(nameVariances, (variance) => { + return { + givenName: variance.given_name, + familyName: variance.family_name } - const pageSize = this.ds.getRequestPageSize().valueOf() - const totalResults = harvestSet.totalResults.valueOf() - - sessionState = harvestSet.sessionState + })) + } + await pMap (normedNameVariances, async (name) => { + const currentHarvestSets: HarvestSet[] = [] + let searchPerson = _.cloneDeep(person) + try { + searchPerson = _.set(searchPerson, 'familyName', name.familyName) + searchPerson = _.set(searchPerson, 'givenName', name.givenName) + searchPerson = _.set(searchPerson, 'givenNameInitial', name.givenName[0]) + } catch (error) { + console.log(`Error on name variance: ${JSON.stringify(name, null, 2)}`) + throw error + } + let offset = 0 + // have to alias this because nested this call below makes this undefined + let thisHarvester = this + + let sessionState = {} + await wait(thisHarvester.ds.getDataSourceConfig().requestInterval) + harvestSet = await this.fetchPublications(searchPerson, harvestBy, sessionState, offset, searchStartDate, searchEndDate) + if (harvestSet) { + currentHarvestSets.push(harvestSet) + } + const pageSize = this.ds.getRequestPageSize().valueOf() + const totalResults = harvestSet.totalResults.valueOf() - if (totalResults > pageSize){ - let numberOfRequests = parseInt(`${totalResults / pageSize}`) //convert to an integer to drop any decimal - //if no remainder subtract one since already did one call - if ((totalResults % pageSize) <= 0) { - numberOfRequests -= 1 - } + sessionState = harvestSet.sessionState - // make variable since 'this' reference scope gets confused inside loops below - const currentDS = this.ds - - //loop to get the result of the results - console.log(`Making ${numberOfRequests} requests for name variation ${searchPerson.familyName}, ${searchPerson.givenNameInitial}`) - await pTimes (numberOfRequests, async function (index) { - await wait(currentDS.getDataSourceConfig().requestInterval) - if (offset + pageSize < totalResults){ - offset += pageSize - } else { - offset += totalResults - offset - } - harvestSet = await thisHarvester.fetchPublications(searchPerson, harvestBy, sessionState, offset, searchStartDate, searchEndDate) - if (harvestSet) { - currentHarvestSets.push(harvestSet) - } - }, { concurrency: 1}) - } - // check total retrieved result matches what was returned - let totalRetrieved = 0 - _.each (currentHarvestSets, (harvestSet) => { - totalRetrieved += harvestSet.sourcePublications.length - }) - if (totalRetrieved < totalResults) { - throw `All expected results not returned for ${searchPerson.familyName}, ${searchPerson.givenName}, expected: ${totalResults} actual: ${totalRetrieved} start date: ${searchPerson.startDate} and end date ${searchPerson.endDate}` - } else { - console.log(`Retrieved (${totalRetrieved} of ${totalResults}) expected results for ${searchPerson.familyName}, ${searchPerson.givenName} start date: ${searchPerson.startDate} and end date ${searchPerson.endDate}`) - // _.each (currentHarvestSets, (harvestSet: HarvestSet) => { - // _.each (harvestSet.normedPublications, (pub: NormedPublication) => { - // console.log(`doi:${pub.doi} Normed Pub search person: ${JSON.stringify(pub.searchPerson, null, 2)}`) - // }) - // }) - // console.log(`Normed pubs`) + if (totalResults > pageSize){ + let numberOfRequests = parseInt(`${totalResults / pageSize}`) //convert to an integer to drop any decimal + //if no remainder subtract one since already did one call + if ((totalResults % pageSize) <= 0) { + numberOfRequests -= 1 } - harvestSets = _.concat(harvestSets, currentHarvestSets) - }, { concurrency: 1 }) - } else { - console.log(`Warning: Skipping harvest of '${person.familyName}, ${person.givenName}' because person start date: ${person.startDate} and end date ${person.endDate} not within search start date ${searchStartDate} and end date ${searchEndDate}.)`) - } + + // make variable since 'this' reference scope gets confused inside loops below + const currentDS = this.ds + + //loop to get the result of the results + console.log(`Making ${numberOfRequests} requests for name variation ${searchPerson.familyName}, ${searchPerson.givenNameInitial}`) + await pTimes (numberOfRequests, async function (index) { + await wait(currentDS.getDataSourceConfig().requestInterval) + if (offset + pageSize < totalResults){ + offset += pageSize + } else { + offset += totalResults - offset + } + harvestSet = await thisHarvester.fetchPublications(searchPerson, harvestBy, sessionState, offset, searchStartDate, searchEndDate) + if (harvestSet) { + currentHarvestSets.push(harvestSet) + } + }, { concurrency: 1}) + } + // check total retrieved result matches what was returned + let totalRetrieved = 0 + _.each (currentHarvestSets, (harvestSet) => { + totalRetrieved += harvestSet.sourcePublications.length + }) + if (totalRetrieved < totalResults) { + throw `All expected results not returned for ${searchPerson.familyName}, ${searchPerson.givenName}, expected: ${totalResults} actual: ${totalRetrieved} start date: ${searchPerson.startDate} and end date ${searchPerson.endDate}` + } else { + console.log(`Retrieved (${totalRetrieved} of ${totalResults}) expected results for ${searchPerson.familyName}, ${searchPerson.givenName} start date: ${searchPerson.startDate} and end date ${searchPerson.endDate}`) + // _.each (currentHarvestSets, (harvestSet: HarvestSet) => { + // _.each (harvestSet.normedPublications, (pub: NormedPublication) => { + // console.log(`doi:${pub.doi} Normed Pub search person: ${JSON.stringify(pub.searchPerson, null, 2)}`) + // }) + // }) + // console.log(`Normed pubs`) + } + harvestSets = _.concat(harvestSets, currentHarvestSets) + }, { concurrency: 1 }) + // } else { + // console.log(`Warning: Skipping harvest of '${person.familyName}, ${person.givenName}' because person start date: ${person.startDate} and end date ${person.endDate} not within search start date ${searchStartDate} and end date ${searchEndDate}.)`) + // } } catch (error) { console.log(error) const errorMessage = `Error on get papers for author: ${person.familyName}, ${person.givenName}: ${error}` diff --git a/ingest/modules/queryNormalizedPeople.ts b/ingest/modules/queryNormalizedPeople.ts index cefb9c2c..f0d52ead 100644 --- a/ingest/modules/queryNormalizedPeople.ts +++ b/ingest/modules/queryNormalizedPeople.ts @@ -1,5 +1,5 @@ import readPersons from '../../client/src/gql/readPersons' -import readPersonsByYear from '../../client/src/gql/readPersonsByYear' +import readPersonsByYearAllCenters from '../../client/src/gql/readPersonsByYearAllCenters' import readCenterMembers from '../../client/src/gql/readCenterMembers' import _ from 'lodash' import { ApolloClient } from 'apollo-client' @@ -98,7 +98,7 @@ export async function getAllSimplifiedPersons (client: ApolloClient) : Promise> { - const queryResult = await client.query(readPersonsByYear(year)) + const queryResult = await client.query(readPersonsByYearAllCenters(year)) return mapToSimplifiedPeople(queryResult.data.persons) } @@ -108,6 +108,6 @@ export async function getAllCenterMembers(client: ApolloClient) : Promise> { - const queryResult = await client.query(readPersonsByYear(year)) + const queryResult = await client.query(readPersonsByYearAllCenters(year)) return mapToNormedPersons(queryResult.data.persons) } \ No newline at end of file From fcbdc4020b0e584a17cc82ea64d1e4f47322c3c9 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 16 Jul 2021 11:09:38 -0400 Subject: [PATCH 02/43] partially done getnormedpubs in PubMedDataSource --- ingest/modules/pubmedDataSource.ts | 47 ++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/ingest/modules/pubmedDataSource.ts b/ingest/modules/pubmedDataSource.ts index bd41fb27..f645207f 100644 --- a/ingest/modules/pubmedDataSource.ts +++ b/ingest/modules/pubmedDataSource.ts @@ -54,10 +54,53 @@ export class PubMedDataSource implements DataSource { return cslStyleAuthors } + // return map of identifier type to id + getResourceIdentifiers (resourceIdentifiers) { + console.log(`Keying resource identifiers by type: ${JSON.stringify(resourceIdentifiers, null,2)}`) + return _.keyBy(resourceIdentifiers, 'resourceIdentifierType') + } + // returns an array of normalized publication objects given ones retrieved fron this datasource getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[]{ - // TODO - return [] + let normedPubs = [] + const mappedOverObject = _.each(sourcePublications, (pub) => { + const title = pub.title; + // console.log(`Pubmed pub is: ${JSON.stringify(jsonObj, null, 2)}`) + // console.log(`Before ubmed pub is: ${JSON.stringify(beforeJsonObj, null, 2)}`) + + const identifiers = this.getResourceIdentifiers(pub.resourceIdentifiers) + // console.log(`Processing Pub: ${JSON.stringify(pub, null, 2)}`) + // console.log(`Found Resource Identifiers for Title: ${title} ids: ${JSON.stringify(identifiers, null, 2)}`) + // let creators = '' + // // const mappedData = await pMap(pub.creators, async (creator, index) => { + + // // if (index > 0) { + // // creators = `${creators};` + // // } + // // creators = `${creators}${creator.familyName}, ${creator.givenName}` + // // }, { concurrency: 1 }); + + // const parsedName = await nameParser({ + // name: `${pub.creators[0].givenName} ${pub.creators[0].familyName}`, + // reduceMethod: 'majority', + // }); + + let doi = identifiers.doi ? identifiers.doi.resourceIdentifier : '' + let pubmedId = identifiers.pubmed ? identifiers.pubmed.resourceIdentifier: '' + console.log(`Creating normed pub for doi: ${doi} pubmed id: ${pubmedId}`) + // update to be part of NormedPublication + let normedPub = { + title: title, + journalTitle: '', + doi: doi, + publicationDate: pub.publicationYear, + datasourceName: 'PubMed', + sourceId: pubmedId, + sourceMetadata: pub + } + normedPubs.push(normedPub) + }) + return normedPubs } //returns a machine readable string version of this source From ab213e43b42950d4b3524cc249938804be609063 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 16 Jul 2021 11:11:38 -0400 Subject: [PATCH 03/43] fix pubmed handling of person dates to include --- ingest/fetchPubmedDataByAuthor.ts | 4 ++-- ingest/modules/normedPublication.ts | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ingest/fetchPubmedDataByAuthor.ts b/ingest/fetchPubmedDataByAuthor.ts index 5db4f907..b3429d4b 100644 --- a/ingest/fetchPubmedDataByAuthor.ts +++ b/ingest/fetchPubmedDataByAuthor.ts @@ -19,7 +19,7 @@ import moment from 'moment' import dotenv from 'dotenv' import resolve from 'path' import fetch from 'node-fetch' -import readPersonsByYear from '../client/src/gql/readPersonsByYear' +import readPersonsByYearAllCenters from '../client/src/gql/readPersonsByYearAllCenters' import { randomWait, wait } from './units/randomWait' dotenv.config({ @@ -187,7 +187,7 @@ async function getFileData(filePath){ } async function getSimplifiedPersons(year) { - const queryResult = await client.query(readPersonsByYear(year)) + const queryResult = await client.query(readPersonsByYearAllCenters(year)) const simplifiedPersons = _.map(queryResult.data.persons, (person) => { return { diff --git a/ingest/modules/normedPublication.ts b/ingest/modules/normedPublication.ts index 0bbb7de6..238a0579 100644 --- a/ingest/modules/normedPublication.ts +++ b/ingest/modules/normedPublication.ts @@ -97,7 +97,8 @@ export default class NormedPublication { } public static getSourceMetadataFileName(pub: NormedPublication): string { - return `${pub.datasourceName}_${pub.sourceId.replace(/\//g, '_')}.json` + const fileName = `${pub.datasourceName}_${pub.sourceId.replace(/\//g, '_')}.json` + return fileName } public static async writeSourceMetadataToJSON(pubs: NormedPublication[], dataDir) { @@ -109,6 +110,7 @@ export default class NormedPublication { const filePath = path.join(jsonFileDir, NormedPublication.getSourceMetadataFileName(pub)) const sourceMetadata = pub.sourceMetadata if (sourceMetadata) { + console.log(`Writing source metadata file: ${filePath}`) await writeToJSONFile(sourceMetadata, filePath) } }, { concurrency: 1}) From 11aaac6dd152af53577847add889e2c4959f777b Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 16 Jul 2021 11:12:30 -0400 Subject: [PATCH 04/43] pubmed write source_metadata to normed pub files --- ingest/joinAuthorPubmedPubs.js | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/ingest/joinAuthorPubmedPubs.js b/ingest/joinAuthorPubmedPubs.js index 1ffbada5..cce8f5e4 100644 --- a/ingest/joinAuthorPubmedPubs.js +++ b/ingest/joinAuthorPubmedPubs.js @@ -74,6 +74,7 @@ async function mapAuthorFiles (filename) { normedPubCSV = _.set(normedPubCSV, 'authorPosition', 1) normedPubCSV = _.set(normedPubCSV, 'isFirstAuthor', true) normedPubCSV = _.set(normedPubCSV, 'isLastAuthor', (pub.creators.length - 1) === 1) + normedPubCSV = _.set(normedPubCSV, 'source_metadata', normedPub.sourceMetadata) return normedPubCSV // return mappedData; }, { concurrency: 1 }); @@ -112,17 +113,31 @@ async function go() { } await pMap(batches, async (batch, index) => { - await writeCsv({ - path: `${pubmedDataDir}pubmedPubsByAuthor.${moment().format('YYYYMMDDHHmmss')}_${index}.csv`, - data: batch - }); const objectToCSVMap = NormedPublication.loadNormedPublicationObjectToCSVMap() // write source metadata to disk - await pMap(batch, async (pub) => { - NormedPublication.writeSourceMetadataToJSON([NormedPublication.getNormedPublicationObjectFromCSVRow(pub, objectToCSVMap)], pubmedDataDir) + await pMap(batch, async (pub, index) => { + const normedPub = { + datasourceName: 'PubMed', + sourceId: pub.source_id, + sourceMetadata: pub.source_metadata + } + await NormedPublication.writeSourceMetadataToJSON([normedPub], pubmedDataDir) + }) + + // remove sourceMetadata from what is written to csv + let csvBatch = [] + _.each(batch, (pub) => { + csvBatch.push(_.omit(pub, 'source_metadata')) }) + + const csvFilePath = `${pubmedDataDir}pubmedPubsByAuthor.${moment().format('YYYYMMDDHHmmss')}_${index}.csv` + console.log(`Writing csv of pub list to: ${csvFilePath}`) + await writeCsv({ + path: csvFilePath, + data: csvBatch + }); }, {concurrency: 1}) } From a1818cdc0d0a38669fb5998bb92d1e6541058f86 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 16 Jul 2021 15:22:42 -0400 Subject: [PATCH 05/43] added missing semantic scholar id to center query --- client/src/gql/readPersonsByYearAllCenters.js | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/gql/readPersonsByYearAllCenters.js b/client/src/gql/readPersonsByYearAllCenters.js index f5f18b44..568469a2 100644 --- a/client/src/gql/readPersonsByYearAllCenters.js +++ b/client/src/gql/readPersonsByYearAllCenters.js @@ -30,6 +30,7 @@ export default function readPersonsByYearAllCenters (year) { family_name start_date end_date + semantic_scholar_id institution { name } From 994e44cfbbc2ab2b5c0e948aec5613a3ad2c4b09 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 19 Jul 2021 21:54:22 -0400 Subject: [PATCH 06/43] convert to multiple semantic ids per person --- client/src/gql/readCenterMembers.js | 2 +- client/src/gql/readPersonsByYearAllCenters.js | 2 +- .../down.yaml | 3 + .../up.yaml | 3 + .../down.yaml | 7 ++ .../up.yaml | 3 + .../down.yaml | 3 + .../up.yaml | 4 + .../down.yaml | 6 ++ .../up.yaml | 6 ++ ingest/fetchSemanticScholarAuthorData.ts | 20 ++--- ingest/gql/updatePersonSemanticScholarIds.js | 24 ++++++ ingest/loadAuthorAttributes.ts | 15 ++-- ingest/modules/normedPerson.ts | 2 +- ingest/modules/queryNormalizedPeople.ts | 6 +- ingest/modules/semanticScholarDataSource.ts | 86 ++++++++++--------- .../test/semanticScholarDataSource.test.ts | 4 +- 17 files changed, 131 insertions(+), 65 deletions(-) create mode 100644 hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml create mode 100644 hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml create mode 100644 hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/down.yaml create mode 100644 hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/up.yaml create mode 100644 hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml create mode 100644 hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml create mode 100644 hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/down.yaml create mode 100644 hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/up.yaml create mode 100644 ingest/gql/updatePersonSemanticScholarIds.js diff --git a/client/src/gql/readCenterMembers.js b/client/src/gql/readCenterMembers.js index 35ec4a17..771ddb9d 100644 --- a/client/src/gql/readCenterMembers.js +++ b/client/src/gql/readCenterMembers.js @@ -16,7 +16,7 @@ export default function readCenterMembers () { family_name start_date end_date - semantic_scholar_id + semantic_scholar_ids institution { name } diff --git a/client/src/gql/readPersonsByYearAllCenters.js b/client/src/gql/readPersonsByYearAllCenters.js index 568469a2..41f8fa87 100644 --- a/client/src/gql/readPersonsByYearAllCenters.js +++ b/client/src/gql/readPersonsByYearAllCenters.js @@ -30,7 +30,7 @@ export default function readPersonsByYearAllCenters (year) { family_name start_date end_date - semantic_scholar_id + semantic_scholar_ids institution { name } diff --git a/hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml b/hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml new file mode 100644 index 00000000..cef6ee65 --- /dev/null +++ b/hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."persons" DROP COLUMN "semantic_scholar_ids"; + type: run_sql diff --git a/hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml b/hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml new file mode 100644 index 00000000..57d2dd6c --- /dev/null +++ b/hasura/migrations/1626721767887_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."persons" ADD COLUMN "semantic_scholar_ids" jsonb NULL; + type: run_sql diff --git a/hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/down.yaml b/hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/down.yaml new file mode 100644 index 00000000..0a17e695 --- /dev/null +++ b/hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/down.yaml @@ -0,0 +1,7 @@ +- args: + sql: ALTER TABLE "public"."persons" ADD COLUMN "semantic_scholar_ids" jsonb + type: run_sql +- args: + sql: ALTER TABLE "public"."persons" ALTER COLUMN "semantic_scholar_ids" DROP NOT + NULL + type: run_sql diff --git a/hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/up.yaml b/hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/up.yaml new file mode 100644 index 00000000..a7a6bf68 --- /dev/null +++ b/hasura/migrations/1626724427810_alter_table_public_persons_drop_column_semantic_scholar_ids/up.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."persons" DROP COLUMN "semantic_scholar_ids" CASCADE + type: run_sql diff --git a/hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml b/hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml new file mode 100644 index 00000000..cef6ee65 --- /dev/null +++ b/hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/down.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."persons" DROP COLUMN "semantic_scholar_ids"; + type: run_sql diff --git a/hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml b/hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml new file mode 100644 index 00000000..a1bbe4b7 --- /dev/null +++ b/hasura/migrations/1626724574879_alter_table_public_persons_add_column_semantic_scholar_ids/up.yaml @@ -0,0 +1,4 @@ +- args: + sql: ALTER TABLE "public"."persons" ADD COLUMN "semantic_scholar_ids" int2vector + NULL; + type: run_sql diff --git a/hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/down.yaml b/hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/down.yaml new file mode 100644 index 00000000..044fe01c --- /dev/null +++ b/hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/down.yaml @@ -0,0 +1,6 @@ +- args: + sql: ALTER TABLE "public"."persons" ALTER COLUMN "semantic_scholar_ids" TYPE ARRAY; + type: run_sql +- args: + sql: COMMENT ON COLUMN "public"."persons"."semantic_scholar_ids" IS E'null' + type: run_sql diff --git a/hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/up.yaml b/hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/up.yaml new file mode 100644 index 00000000..86525e1c --- /dev/null +++ b/hasura/migrations/1626725098496_alter_table_public_persons_alter_column_semantic_scholar_ids/up.yaml @@ -0,0 +1,6 @@ +- args: + sql: ALTER TABLE "public"."persons" ALTER COLUMN "semantic_scholar_ids" TYPE text; + type: run_sql +- args: + sql: COMMENT ON COLUMN "public"."persons"."semantic_scholar_ids" IS E'' + type: run_sql diff --git a/ingest/fetchSemanticScholarAuthorData.ts b/ingest/fetchSemanticScholarAuthorData.ts index 298811c8..905dc63c 100644 --- a/ingest/fetchSemanticScholarAuthorData.ts +++ b/ingest/fetchSemanticScholarAuthorData.ts @@ -91,7 +91,7 @@ async function main (): Promise { personCounter += 1 const person = persons[0] const personId = person['id'] - if (person.sourceIds && person.sourceIds.semanticScholarId || possibleAuthorIdsByPersonId[`${personId}`]) { + if (person.sourceIds && person.sourceIds.semanticScholarIds || possibleAuthorIdsByPersonId[`${personId}`]) { console.log(`Getting papers for ${person.familyName}, ${person.givenName}`) // run for each name plus name variance, put name variance second in case undefined // let searchNames = _.concat([{given_name: person.firstName, family_name: person.lastName }], person.nameVariances) @@ -102,20 +102,18 @@ async function main (): Promise { person.sourceIds = {} } - if (person.sourceIds && person.sourceIds.semanticScholarId) { - semanticScholarIds.push(person.sourceIds.semanticScholarId) + if (person.sourceIds && person.sourceIds.semanticScholarIds) { + semanticScholarIds = person.sourceIds.semanticScholarIds } // } else if (possibleAuthorIdsByPersonId[`${personId}`]) { // semanticScholarIds = _.concat(semanticScholarIds, possibleAuthorIdsByPersonId[`${personId}`]) // } - await pMap(semanticScholarIds, async (scholarId) => { - let harvestPerson = _.clone(person) - harvestPerson.sourceIds.semanticScholarId = scholarId - const harvestPersons = [harvestPerson] - await semanticScholarHarvester.harvestToCsv(resultsDir, persons, HarvestOperation.QUERY_BY_AUTHOR_ID, getDateObject(`${year}-01-01`), getDateObject(`${year}-12-31`), `${person.familyName}_${person.givenName}`) - // await pMap(searchNames, async (searchName) => { - await wait(1500) - }, { concurrency: 1}) + let harvestPerson = _.clone(person) + harvestPerson.sourceIds.semanticScholarIds = semanticScholarIds + const harvestPersons = [harvestPerson] + await semanticScholarHarvester.harvestToCsv(resultsDir, persons, HarvestOperation.QUERY_BY_AUTHOR_ID, getDateObject(`${year}-01-01`), getDateObject(`${year}-12-31`), `${person.familyName}_${person.givenName}`) + // await pMap(searchNames, async (searchName) => { + await wait(1500) // }, { concurrency: 1}) succeededAuthors = _.concat(succeededAuthors, persons) diff --git a/ingest/gql/updatePersonSemanticScholarIds.js b/ingest/gql/updatePersonSemanticScholarIds.js new file mode 100644 index 00000000..d5c6eced --- /dev/null +++ b/ingest/gql/updatePersonSemanticScholarIds.js @@ -0,0 +1,24 @@ +import gql from 'graphql-tag' + +export default function updatePersonSemanticScholarIds (id, semanticScholarIds) { + return { + mutation: gql` + mutation MyMutation($id: Int!, $semantic_scholar_ids: String!) { + update_persons(where: {id: {_eq: $id}}, _set: {semantic_scholar_ids: $semantic_scholar_ids}) { + returning { + id + given_name + family_name + start_date + end_date, + semantic_scholar_ids + } + } + } + `, + variables: { + id, + semantic_scholar_ids: JSON.stringify(semanticScholarIds) + } + } +} diff --git a/ingest/loadAuthorAttributes.ts b/ingest/loadAuthorAttributes.ts index 75c69f99..b82a11b2 100644 --- a/ingest/loadAuthorAttributes.ts +++ b/ingest/loadAuthorAttributes.ts @@ -7,7 +7,7 @@ import pMap from 'p-map' import _, { update } from 'lodash' import { command as loadCsv } from './units/loadCsv' import readPersons from '../client/src/gql/readPersons' -import updatePersonSemanticScholarId from './gql/updatePersonSemanticScholarId' +import updatePersonSemanticScholarIds from './gql/updatePersonSemanticScholarIds' import { __EnumValue } from 'graphql' import { getAllSimplifiedPersons, getNameKey } from './modules/queryNormalizedPeople' @@ -61,14 +61,19 @@ async function main (): Promise { }) const nameKey = getNameKey(author[_.keys(author)[familyNameIndex]], author['given_name']) // console.log(`Person map is: ${JSON.stringify(_.keys(personMap).length, null, 2)}`) - // console.log(`Name key is: ${nameKey}`) + console.log(`Name key is: ${nameKey}`) const personId = personMap[nameKey][0].id if (personMap[nameKey]) { if (author['semantic_scholar_id']){ if (!updateSourceIds[personId]){ updateSourceIds[personId] = {} } - updateSourceIds[personId]['semanticScholarId'] = author['semantic_scholar_id'] + // add multiples as array delimited by ';' + let semanticScholarIds = _.split(author['semantic_scholar_id'], ';') + semanticScholarIds = _.map(semanticScholarIds, (id) => { + return _.trim(id) + }) + updateSourceIds[personId]['semanticScholarIds'] = semanticScholarIds } if (author['name_variances']) { const existingNameVariances = personMap[nameKey][0].nameVariances @@ -130,8 +135,8 @@ async function main (): Promise { let updatedSourceIds = 0 await pMap(_.keys(updateSourceIds), async (updatePersonId) => { const sourceIds = updateSourceIds[updatePersonId] - if (sourceIds['semanticScholarId']){ - const resultUpdateScholarId = await client.mutate(updatePersonSemanticScholarId(updatePersonId, sourceIds['semanticScholarId'])) + if (sourceIds['semanticScholarIds']){ + const resultUpdateScholarId = await client.mutate(updatePersonSemanticScholarIds(updatePersonId, sourceIds['semanticScholarIds'])) updatedSourceIds += resultUpdateScholarId.data.update_persons.returning.length } }, { concurrency: 1 }) diff --git a/ingest/modules/normedPerson.ts b/ingest/modules/normedPerson.ts index 14b27734..d24d9dfb 100644 --- a/ingest/modules/normedPerson.ts +++ b/ingest/modules/normedPerson.ts @@ -13,7 +13,7 @@ export default class NormedPerson { nameVariances?: [] sourceIds: { scopusAffiliationId?: string, - semanticScholarId?: string + semanticScholarIds?: string[] } // ------ end declare properties used when using NormedPerson like an interface diff --git a/ingest/modules/queryNormalizedPeople.ts b/ingest/modules/queryNormalizedPeople.ts index f0d52ead..69878285 100644 --- a/ingest/modules/queryNormalizedPeople.ts +++ b/ingest/modules/queryNormalizedPeople.ts @@ -32,7 +32,7 @@ interface NormedCenterMember { endDate: Date sourceIds: { scopusAffiliationId?: string, - semanticScholarId?: string + semanticScholarIds?: [] } } @@ -46,7 +46,7 @@ function mapToNormedPersons(people: Array) : Array { startDate: getDateObject(person.start_date), endDate: getDateObject(person.end_date), nameVariances: person.persons_namevariances, - sourceIds: { semanticScholarId: person.semantic_scholar_id } + sourceIds: { semanticScholarIds: JSON.parse(person.semantic_scholar_ids) } } return np }) @@ -81,7 +81,7 @@ function mapToCenterMembers(members: Array) : NormedCenterMember[] { givenNameInitial: _.toLower(member.person.given_name[0]), startDate: member.start_date, endDate: member.end_date, - sourceIds: { semanticScholarId: member.person.semantic_scholar_id } + sourceIds: { semanticScholarIds: member.person.semantic_scholar_ids } } normedMembers.push(normedMember) }) diff --git a/ingest/modules/semanticScholarDataSource.ts b/ingest/modules/semanticScholarDataSource.ts index 0250354e..f824951e 100644 --- a/ingest/modules/semanticScholarDataSource.ts +++ b/ingest/modules/semanticScholarDataSource.ts @@ -24,7 +24,7 @@ export class SemanticScholarDataSource implements DataSource { // return object with query request params getAuthorQuery(person: NormedPerson, startDate?: Date, endDate?: Date){ if (person.sourceIds) { - return `authorId:${person.sourceIds.semanticScholarId}` + return `authorId:${person.sourceIds.semanticScholarIds}` } else { return undefined } @@ -36,7 +36,7 @@ export class SemanticScholarDataSource implements DataSource { // assumes that if only one of startDate or endDate provided it would always be startDate first and then have endDate undefined async getPublicationsByAuthorId(person: NormedPerson, sessionState: {}, offset: Number, startDate: Date, endDate?: Date): Promise { - let finalTotalResults: Number + let finalTotalResults = 0 let publications = [] let skippedPublications = 0 @@ -44,58 +44,62 @@ export class SemanticScholarDataSource implements DataSource { const minPublicationYear = (startDate ? startDate.getFullYear() : undefined) const maxPublicationYear = (endDate ? endDate.getFullYear() : undefined) - if (!person.sourceIds || !person.sourceIds.semanticScholarId) { + if (!person.sourceIds || !person.sourceIds.semanticScholarIds) { throw (`Semantic Scholar Id not defined for Person: ${JSON.stringify(person)}`) } - const authorId = person.sourceIds.semanticScholarId + const authorIds = person.sourceIds.semanticScholarIds // need to make sure date string in correct format - const results = await this.fetchSemanticScholarAuthorData(this.dsConfig.pageSize, offset, authorId) - // console.log (`semantic scholar results are: ${_.keys(results['papers'])}`) - if (results && results['papers']){ - - const totalResults = Number.parseInt(results['papers'].length) - finalTotalResults = totalResults - if (totalResults > 0){ - // fetch metadata for each paper - await wait(this.dsConfig.requestInterval) - const papers = results['papers'] - await pMap (papers, async (paper, index) => { - const paperId = paper['paperId'] - let paperYear = paper['year'] - let skipPublication = false - if (paperYear) { - paperYear = Number.parseInt(paperYear) - if (minPublicationYear) { - if (paperYear < minPublicationYear) { - skipPublication = true - } else if (maxPublicationYear && paperYear > maxPublicationYear) { - skipPublication = true + // fetch for each possible id expecting there could be more than one + await pMap(authorIds, async (authorId) => { + await wait(this.dsConfig.requestInterval) + const results = await this.fetchSemanticScholarAuthorData(this.dsConfig.pageSize, offset, authorId) + // console.log (`semantic scholar results are: ${_.keys(results['papers'])}`) + if (results && results['papers']){ + + const totalResults = Number.parseInt(results['papers'].length) + finalTotalResults += totalResults + if (totalResults > 0){ + // fetch metadata for each paper + await wait(this.dsConfig.requestInterval) + const papers = results['papers'] + await pMap (papers, async (paper, index) => { + const paperId = paper['paperId'] + let paperYear = paper['year'] + let skipPublication = false + if (paperYear) { + paperYear = Number.parseInt(paperYear) + if (minPublicationYear) { + if (paperYear < minPublicationYear) { + skipPublication = true + } else if (maxPublicationYear && paperYear > maxPublicationYear) { + skipPublication = true + } } } - } - if (!skipPublication) { - await wait(this.dsConfig.requestInterval) - console.log(`Fetching paper metadata (${(index + 1)} of ${totalResults}) for author: ${person.familyName}, ${person.givenName}`) - const paperMetadata = await this.fetchSemanticScholarPaperData(paperId) - publications.push(paperMetadata) - } else { - console.log(`Skipping paper metadata (${(index + 1)} of ${totalResults}) for author: ${person.familyName}, ${person.givenName} with publication year: ${paperYear}`) - skippedPublications += 1 - finalTotalResults = finalTotalResults.valueOf() - 1 - } - }, { concurrency: 1 }) + if (!skipPublication) { + await wait(this.dsConfig.requestInterval) + console.log(`Fetching paper metadata (${(index + 1)} of ${totalResults}) for author: ${person.familyName}, ${person.givenName}`) + const paperMetadata = await this.fetchSemanticScholarPaperData(paperId) + publications.push(paperMetadata) + } else { + console.log(`Skipping paper metadata (${(index + 1)} of ${totalResults}) for author: ${person.familyName}, ${person.givenName} with publication year: ${paperYear}`) + skippedPublications += 1 + finalTotalResults = finalTotalResults.valueOf() - 1 + } + }, { concurrency: 1 }) + } + } else { + finalTotalResults += 0 } - } else { - finalTotalResults = 0 - } + }, { concurrency: 1}) console.log(`Fetched ${(finalTotalResults)} publications and Skipped ${skippedPublications} publications outside of publication target range for author: ${person.familyName}, ${person.givenName}`) const result: HarvestSet = { sourceName: this.getSourceName(), searchPerson: person, - query: `authorId:${authorId}`, + query: `authorIds:${authorIds}`, sourcePublications: publications, offset: offset, pageSize: Number.parseInt(this.dsConfig.pageSize), diff --git a/ingest/modules/test/semanticScholarDataSource.test.ts b/ingest/modules/test/semanticScholarDataSource.test.ts index bc2e3277..bd2ec3bd 100644 --- a/ingest/modules/test/semanticScholarDataSource.test.ts +++ b/ingest/modules/test/semanticScholarDataSource.test.ts @@ -123,7 +123,7 @@ beforeAll(async () => { givenName: 'Jun', startDate: getDateObject('2019-01-01'), sourceIds: { - semanticScholarId: '46276642' + semanticScholarIds: ['46276642'] }, endDate: undefined } @@ -173,7 +173,7 @@ test('test Semantic Scholar harvester.fetchPublications by Author Id', async () test('test Semantic Scholar getAuthorQuery', () => { expect.hasAssertions() const authorQuery = semanticScholarDS.getAuthorQuery(defaultNormedPerson) - expect(authorQuery).toEqual(`authorId:${defaultNormedPerson.sourceIds.semanticScholarId}`) + expect(authorQuery).toEqual(`authorId:${defaultNormedPerson.sourceIds.semanticScholarIds}`) }) test('test Semantic Scholar semanticScholarDataSource.getCoauthors', async () => { From 6e6a9845dc25623c06d4c8aaf29d3cdb0977d510 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 20 Jul 2021 15:38:10 -0400 Subject: [PATCH 07/43] add given name mismatch confidence checking --- .../down.yaml | 3 ++ .../up.yaml | 4 ++ .../down.yaml | 23 +++++++++ .../up.yaml | 24 ++++++++++ .../down.yaml | 21 +++++++++ .../up.yaml | 22 +++++++++ .../down.yaml | 23 +++++++++ .../up.yaml | 24 ++++++++++ .../down.yaml | 1 + .../up.yaml | 5 ++ hasura/migrations/metadata.yaml | 13 +++-- ingest/gql/readConfidenceTypes.js | 1 + ingest/modules/calculateConfidence.ts | 47 ++++++++++++++++--- 13 files changed, 199 insertions(+), 12 deletions(-) create mode 100644 hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/down.yaml create mode 100644 hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/up.yaml create mode 100644 hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/down.yaml create mode 100644 hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/up.yaml create mode 100644 hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/down.yaml create mode 100644 hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/up.yaml create mode 100644 hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/down.yaml create mode 100644 hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/up.yaml create mode 100644 hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/down.yaml create mode 100644 hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/up.yaml diff --git a/hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/down.yaml b/hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/down.yaml new file mode 100644 index 00000000..986c78d9 --- /dev/null +++ b/hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/down.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."confidence_type" DROP COLUMN "stop_testing_if_passed"; + type: run_sql diff --git a/hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/up.yaml b/hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/up.yaml new file mode 100644 index 00000000..7c29f348 --- /dev/null +++ b/hasura/migrations/1626792544531_alter_table_public_confidence_type_add_column_stop_testing_if_passed/up.yaml @@ -0,0 +1,4 @@ +- args: + sql: ALTER TABLE "public"."confidence_type" ADD COLUMN "stop_testing_if_passed" + boolean NULL; + type: run_sql diff --git a/hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/down.yaml b/hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/down.yaml new file mode 100644 index 00000000..6710b8bf --- /dev/null +++ b/hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/down.yaml @@ -0,0 +1,23 @@ +- args: + role: user + table: + name: confidence_type + schema: public + type: drop_insert_permission +- args: + permission: + check: {} + columns: + - id + - name + - description + - rank + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: confidence_type + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/up.yaml b/hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/up.yaml new file mode 100644 index 00000000..305ef6b3 --- /dev/null +++ b/hasura/migrations/1626795231985_update_permission_user_public_table_confidence_type/up.yaml @@ -0,0 +1,24 @@ +- args: + role: user + table: + name: confidence_type + schema: public + type: drop_insert_permission +- args: + permission: + check: {} + columns: + - description + - id + - name + - rank + - stop_testing_if_passed + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: confidence_type + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/down.yaml b/hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/down.yaml new file mode 100644 index 00000000..d8630722 --- /dev/null +++ b/hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/down.yaml @@ -0,0 +1,21 @@ +- args: + role: user + table: + name: confidence_type + schema: public + type: drop_select_permission +- args: + permission: + allow_aggregations: true + columns: + - id + - rank + - description + - name + computed_fields: [] + filter: {} + role: user + table: + name: confidence_type + schema: public + type: create_select_permission diff --git a/hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/up.yaml b/hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/up.yaml new file mode 100644 index 00000000..4efd9277 --- /dev/null +++ b/hasura/migrations/1626795245330_update_permission_user_public_table_confidence_type/up.yaml @@ -0,0 +1,22 @@ +- args: + role: user + table: + name: confidence_type + schema: public + type: drop_select_permission +- args: + permission: + allow_aggregations: true + columns: + - description + - id + - name + - rank + - stop_testing_if_passed + computed_fields: [] + filter: {} + role: user + table: + name: confidence_type + schema: public + type: create_select_permission diff --git a/hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/down.yaml b/hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/down.yaml new file mode 100644 index 00000000..4455d477 --- /dev/null +++ b/hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/down.yaml @@ -0,0 +1,23 @@ +- args: + role: user + table: + name: confidence_type + schema: public + type: drop_update_permission +- args: + permission: + columns: + - id + - rank + - description + - name + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: confidence_type + schema: public + type: create_update_permission diff --git a/hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/up.yaml b/hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/up.yaml new file mode 100644 index 00000000..a41f0bf9 --- /dev/null +++ b/hasura/migrations/1626795256618_update_permission_user_public_table_confidence_type/up.yaml @@ -0,0 +1,24 @@ +- args: + role: user + table: + name: confidence_type + schema: public + type: drop_update_permission +- args: + permission: + columns: + - description + - id + - name + - rank + - stop_testing_if_passed + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: confidence_type + schema: public + type: create_update_permission diff --git a/hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/down.yaml b/hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/down.yaml new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/down.yaml @@ -0,0 +1 @@ +[] diff --git a/hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/up.yaml b/hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/up.yaml new file mode 100644 index 00000000..9ef9b925 --- /dev/null +++ b/hasura/migrations/1626795422921_insert_given_name_mismatch_confidence_type/up.yaml @@ -0,0 +1,5 @@ +- args: + cascade: false + sql: | + insert into confidence_type (name, description, rank, stop_testing_if_passed) values ('given_name_mismatch', 'Given name mismatch', 3, true); + type: run_sql diff --git a/hasura/migrations/metadata.yaml b/hasura/migrations/metadata.yaml index b1bfe639..f6355b4e 100644 --- a/hasura/migrations/metadata.yaml +++ b/hasura/migrations/metadata.yaml @@ -355,10 +355,11 @@ tables: set: {} check: {} columns: + - description - id - name - - description - rank + - stop_testing_if_passed select_permissions: - role: user comment: null @@ -366,10 +367,11 @@ tables: allow_aggregations: true computed_fields: [] columns: - - id - - rank - description + - id - name + - rank + - stop_testing_if_passed filter: {} update_permissions: - role: user @@ -377,10 +379,11 @@ tables: permission: set: {} columns: - - id - - rank - description + - id - name + - rank + - stop_testing_if_passed filter: {} delete_permissions: [] event_triggers: [] diff --git a/ingest/gql/readConfidenceTypes.js b/ingest/gql/readConfidenceTypes.js index 1ecabbcf..0857ce72 100644 --- a/ingest/gql/readConfidenceTypes.js +++ b/ingest/gql/readConfidenceTypes.js @@ -9,6 +9,7 @@ export default function readConfidenceTypes () { id name rank + stop_testing_if_passed } } ` diff --git a/ingest/modules/calculateConfidence.ts b/ingest/modules/calculateConfidence.ts index 091a2d95..571605f4 100644 --- a/ingest/modules/calculateConfidence.ts +++ b/ingest/modules/calculateConfidence.ts @@ -428,7 +428,7 @@ export class CalculateConfidence { _.each(_.keys(publicationAuthorMap), (pubLastName) => { // check for a fuzzy match of name variant last names to lastname in pub author list if (this.lastNameMatchFuzzy(pubLastName, 'lastName', nameVariations[nameLastName]) || this.lastNameMatchFuzzy(pubLastName, 'family', nameVariations[nameLastName])){ - //console.log(`Found lastname match pub: ${pubLastName} and variation: ${nameLastName}`) + // console.log(`Found lastname match pub: ${pubLastName} and variation: ${nameLastName}`) // now check for first initial or given name match // split the given name based on spaces @@ -446,7 +446,9 @@ export class CalculateConfidence { console.log(`splitting given parts pubAuthor is: ${JSON.stringify(pubAuthor, null, 2)}`) } if (part && this.nameMatchFuzzy(pubLastName, 'lastName', part.toLowerCase(), firstKey, nameVariations[nameLastName])) { - const testPart = part.replace(/./g,'') + // console.log(`found match for author: ${JSON.stringify(pubAuthor, null, 2)}`) + const testPart = part.replace(/\./g,'') + // console.log(`part is '${part}' Test part is: '${testPart}' failIfOnlyInitialInGivenName is: ${failIfOnlyInitialInGivenName}`) if (!failIfOnlyInitialInGivenName || testPart.length > 1){ (matchedAuthors[pubLastName] || (matchedAuthors[pubLastName] = [])).push(pubAuthor) matched = true @@ -456,7 +458,7 @@ export class CalculateConfidence { // if not matched try matching without breaking it into parts if (!matched && givenParts.length > 1) { if (this.nameMatchFuzzy(pubLastName, 'lastName', pubAuthor['given'], firstKey, nameVariations[nameLastName])) { - const testPart = pubAuthor['given'].replace(/./g,'') + const testPart = pubAuthor['given'].replace(/\./g,'') // only set to true if not failing for 1 character names (i.e., initial) if (!failIfOnlyInitialInGivenName || testPart.length > 1){ (matchedAuthors[pubLastName] || (matchedAuthors[pubLastName] = [])).push(pubAuthor) @@ -475,11 +477,37 @@ export class CalculateConfidence { return this.testAuthorGivenNamePart(author, publicationAuthorMap, true) } - // only call this method if last name matched + // only call this method if last name and initials matched testAuthorGivenName (author, publicationAuthorMap, failIfOnlyInitialInGivenName?) { return this.testAuthorGivenNamePart(author, publicationAuthorMap, false, failIfOnlyInitialInGivenName) } + // only call this method if last name and initials matched + testAuthorGivenNameMismatch (author, publicationAuthorMap) { + // check if initials passed, if initials do not pass then say it is a mismatch + // -- if initials passed then check given name. + // -- if given name length 1 char only or moret than not a match say there is a mismatch + + const testInitialsMatchedAuthors = this.testAuthorGivenNameInitial (author, publicationAuthorMap) + const testInitialsMatch = (testInitialsMatchedAuthors && _.keys(testInitialsMatchedAuthors).length > 0) + if (testInitialsMatch) { + // check if passes with failIfOnlyInitialInGivenName to false so it allows matches if length = 1 + const testInitialsAllowedMatchedAuthors = this.testAuthorGivenNamePart (author, publicationAuthorMap, false, false) + const testInitialsAllowed = (testInitialsAllowedMatchedAuthors && _.keys(testInitialsAllowedMatchedAuthors).length > 0) + // const testInitialsNotAllowed = (testInitialsNotAllowedMatchedAuthors && _.keys(testInitialsNotAllowedMatchedAuthors).length > 0) + if (testInitialsAllowed) { + // console.log(`No given Name mismatch both with initials and without`) + return {} + } else { + console.log(`Given Name mismatch detected initials match, but not other match test author: ${JSON.stringify(author, null, 2)} pub authors: ${JSON.stringify(publicationAuthorMap, null, 2)}`) + return testInitialsMatchedAuthors + } + } else { + // console.log(`Given Name mismatch no initials match, so just returning empty set to ignore this test as only applies when initial match found`) + return {} + } + } + getAuthorsFromSourceMetadata(sourceName, sourceMetadata) { if (_.toLower(sourceName)==='pubmed'){ return _.mapValues(sourceMetadata['creators'], (creator) => { @@ -547,8 +575,10 @@ testAuthorAffiliation (author, publicationAuthorMap, sourceName, sourceMetadata) return matchedAuthors } else if (confidenceType.name === 'given_name_initial') { return this.testAuthorGivenNameInitial(author, publicationAuthorMap) - // } else if (confidenceType.name === 'given_name_mismatch') { - // return this.testAuthorGivenNameMismatch(author, publicationAuthorMap) + } else if (confidenceType.name === 'given_name_mismatch') { + // this one opposite other tests where a set is returned if mismatches are found, setting it true + // console.log('Checking if given name mismatch...') + return this.testAuthorGivenNameMismatch(author, publicationAuthorMap) } else if (confidenceType.name === 'given_name') { return this.testAuthorGivenName(author, publicationAuthorMap, true) } else if (confidenceType.name === 'university_affiliation') { @@ -606,9 +636,12 @@ testAuthorAffiliation (author, publicationAuthorMap, sourceName, sourceMetadata) matchedAuthors[matchedLastName] = currentMatchedAuthors[matchedLastName] } }) + if (confidenceType['stop_testing_if_passed']){ + stopTesting = true + } } }, {concurrency: 3}) - if (_.keys(matchedAuthors).length <= 0){ + if (_.keys(matchedAuthors).length <= 0 || stopTesting){ // stop processing and skip next set of tests stopTesting = true } else { From d81f603fb5aaf9698a131896a6b372e496c1ded5 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 20 Jul 2021 15:54:16 -0400 Subject: [PATCH 08/43] remove mentions of singular semantic_scholar_id --- .../down.yaml | 7 +++++++ .../up.yaml | 3 +++ hasura/migrations/metadata.yaml | 13 ++++++++----- .../gql/readPersonPublicationsConfSetsBySource.js | 2 +- ingest/mineSemanticScholarAuthorIds.ts | 4 ++-- 5 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/down.yaml create mode 100644 hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/up.yaml diff --git a/hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/down.yaml b/hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/down.yaml new file mode 100644 index 00000000..b99624c2 --- /dev/null +++ b/hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/down.yaml @@ -0,0 +1,7 @@ +- args: + sql: ALTER TABLE "public"."persons" ADD COLUMN "semantic_scholar_id" int4 + type: run_sql +- args: + sql: ALTER TABLE "public"."persons" ALTER COLUMN "semantic_scholar_id" DROP NOT + NULL + type: run_sql diff --git a/hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/up.yaml b/hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/up.yaml new file mode 100644 index 00000000..2c433645 --- /dev/null +++ b/hasura/migrations/1626810737736_alter_table_public_persons_drop_column_semantic_scholar_id/up.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."persons" DROP COLUMN "semantic_scholar_id" CASCADE + type: run_sql diff --git a/hasura/migrations/metadata.yaml b/hasura/migrations/metadata.yaml index b1bfe639..f6355b4e 100644 --- a/hasura/migrations/metadata.yaml +++ b/hasura/migrations/metadata.yaml @@ -355,10 +355,11 @@ tables: set: {} check: {} columns: + - description - id - name - - description - rank + - stop_testing_if_passed select_permissions: - role: user comment: null @@ -366,10 +367,11 @@ tables: allow_aggregations: true computed_fields: [] columns: - - id - - rank - description + - id - name + - rank + - stop_testing_if_passed filter: {} update_permissions: - role: user @@ -377,10 +379,11 @@ tables: permission: set: {} columns: - - id - - rank - description + - id - name + - rank + - stop_testing_if_passed filter: {} delete_permissions: [] event_triggers: [] diff --git a/ingest/gql/readPersonPublicationsConfSetsBySource.js b/ingest/gql/readPersonPublicationsConfSetsBySource.js index e5c84ab3..322b4356 100644 --- a/ingest/gql/readPersonPublicationsConfSetsBySource.js +++ b/ingest/gql/readPersonPublicationsConfSetsBySource.js @@ -25,7 +25,7 @@ export default function readPersonPublicationsConfSetsBySource (sourceName) { given_name family_name } - semantic_scholar_id + semantic_scholar_ids } publication { id diff --git a/ingest/mineSemanticScholarAuthorIds.ts b/ingest/mineSemanticScholarAuthorIds.ts index ba5b486a..e0f47074 100644 --- a/ingest/mineSemanticScholarAuthorIds.ts +++ b/ingest/mineSemanticScholarAuthorIds.ts @@ -168,8 +168,8 @@ async function main (): Promise { if (!existingSemanticIds[personPub.person_id]){ existingSemanticIds[personPub.person_id] = [] } - if (personPub.person.semantic_scholar_id) { - existingSemanticIds[personPub.person_id].push(personPub.person.semantic_scholar_id) + if (personPub.person.semantic_scholar_ids) { + existingSemanticIds[personPub.person_id] = _.concat(existingSemanticIds[personPub.person_id], JSON.parse(personPub.person.semantic_scholar_id)) } // now return doi to group by them return personPub.doi From d56a4615682a9514f3e7e5f1de51b481c3a0fa9a Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 20 Jul 2021 16:16:45 -0400 Subject: [PATCH 09/43] add'l semantic_scholar_id fixes --- client/src/gql/readCenterMembers.js | 2 +- client/src/gql/readPersons.js | 2 +- client/src/gql/readPersonsByYear.js | 2 +- client/src/gql/readPersonsByYearAllCenters.js | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/src/gql/readCenterMembers.js b/client/src/gql/readCenterMembers.js index 35ec4a17..771ddb9d 100644 --- a/client/src/gql/readCenterMembers.js +++ b/client/src/gql/readCenterMembers.js @@ -16,7 +16,7 @@ export default function readCenterMembers () { family_name start_date end_date - semantic_scholar_id + semantic_scholar_ids institution { name } diff --git a/client/src/gql/readPersons.js b/client/src/gql/readPersons.js index 170b682f..a01c483f 100644 --- a/client/src/gql/readPersons.js +++ b/client/src/gql/readPersons.js @@ -10,7 +10,7 @@ export default function readPersons () { family_name start_date end_date - semantic_scholar_id + semantic_scholar_ids institution { name } diff --git a/client/src/gql/readPersonsByYear.js b/client/src/gql/readPersonsByYear.js index e0b21350..0d1e8dd0 100644 --- a/client/src/gql/readPersonsByYear.js +++ b/client/src/gql/readPersonsByYear.js @@ -24,7 +24,7 @@ export default function readPersonsByYear (year) { family_name start_date end_date - semantic_scholar_id + semantic_scholar_ids institution { name } diff --git a/client/src/gql/readPersonsByYearAllCenters.js b/client/src/gql/readPersonsByYearAllCenters.js index 568469a2..41f8fa87 100644 --- a/client/src/gql/readPersonsByYearAllCenters.js +++ b/client/src/gql/readPersonsByYearAllCenters.js @@ -30,7 +30,7 @@ export default function readPersonsByYearAllCenters (year) { family_name start_date end_date - semantic_scholar_id + semantic_scholar_ids institution { name } From 53fe0bf031d1eb1df4fdd164d0d11f24f07b9bc3 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 20 Jul 2021 19:35:54 -0400 Subject: [PATCH 10/43] fix recheck_pubs --- client/src/gql/readAllPublicationsCSL.js | 14 ++++++++++++++ ingest/gql/readPublicationsCSL.js | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 client/src/gql/readAllPublicationsCSL.js diff --git a/client/src/gql/readAllPublicationsCSL.js b/client/src/gql/readAllPublicationsCSL.js new file mode 100644 index 00000000..f9907ffc --- /dev/null +++ b/client/src/gql/readAllPublicationsCSL.js @@ -0,0 +1,14 @@ +import gql from 'graphql-tag' +import _ from 'lodash' + +export default function readAllPublicationsCSL () { + return gql` + query MyQuery { + publications { + id + doi + csl + } + } + ` +} diff --git a/ingest/gql/readPublicationsCSL.js b/ingest/gql/readPublicationsCSL.js index 3d3778d8..7fbee730 100644 --- a/ingest/gql/readPublicationsCSL.js +++ b/ingest/gql/readPublicationsCSL.js @@ -4,7 +4,7 @@ export default function readPublicationsCSL () { return { query: gql` query MyQuery { - publications (order_by: {id: desc}){ + publications{ id csl doi From 92470442484bdabd84db7883290ec66c59922472 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Wed, 21 Jul 2021 09:12:21 -0400 Subject: [PATCH 11/43] add tedst code 1 author for rechecking pub matches --- ingest/updatePersonPublicationsMatches.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ingest/updatePersonPublicationsMatches.ts b/ingest/updatePersonPublicationsMatches.ts index 88df9616..42916af1 100644 --- a/ingest/updatePersonPublicationsMatches.ts +++ b/ingest/updatePersonPublicationsMatches.ts @@ -287,6 +287,9 @@ async function findAuthorMatches(testAuthors, confirmedAuthors, doi, csl, source // populate with array of person id's mapped person object let authorMatchesFound = {} + let testAuthors2 = [] + testAuthors2.push(_.find(testAuthors, (testAuthor) => { return testAuthor['id']===1698})) + try { //create map of last name to array of related persons with same last name const personMap = _.transform(testAuthors, function (result, value) { From 934f8911b4326a7b7b01a0387197ec9c492bce1d Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Wed, 21 Jul 2021 20:11:39 -0400 Subject: [PATCH 12/43] update min confidence to 0.4 --- ingest/modules/calculateConfidence.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest/modules/calculateConfidence.ts b/ingest/modules/calculateConfidence.ts index 571605f4..85362ddc 100644 --- a/ingest/modules/calculateConfidence.ts +++ b/ingest/modules/calculateConfidence.ts @@ -734,7 +734,7 @@ testAuthorAffiliation (author, publicationAuthorMap, sourceName, sourceMetadata) console.log('Entering loop 1...') - const minConfidence = 0.45 + const minConfidence = 0.40 await pMap(testAuthors, async (testAuthor) => { console.log(`Confidence Test Author is: ${testAuthor['names'][0]['lastName']}, ${testAuthor['names'][0]['firstName']}`) From b6ea819c7fa379de79eaf23287935eb2537f89d6 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 22 Jul 2021 11:58:14 -0400 Subject: [PATCH 13/43] change crossref use nd filter only if result >300 --- ingest/modules/crossrefDataSource.ts | 56 ++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/ingest/modules/crossrefDataSource.ts b/ingest/modules/crossrefDataSource.ts index b8fd33da..5b11c7dc 100644 --- a/ingest/modules/crossrefDataSource.ts +++ b/ingest/modules/crossrefDataSource.ts @@ -51,22 +51,36 @@ export class CrossRefDataSource implements DataSource { let totalResults: Number let publications = [] - - // need to make sure date string in correct format - const results = await this.fetchCrossRefQuery(this.dsConfig.pageSize, offset, query['query.author'], query['query.affiliation'], query['filter']) - if (results && results['message']['total-results']){ - totalResults = Number.parseInt(results['message']['total-results']) - if (totalResults > 0 && results['message']['items']){ - publications = results['message']['items'] - } + let itemsPerPage = undefined + + const recordLimit = 300 + //first trying running without affiliation and see what results are returned + //if below limit than run without filter, otherwise apply + let affiliation = undefined + let results = await this.fetchCrossRefResults(this.dsConfig.pageSize, offset, query['query.author'], affiliation, query['filter']) + console.log(`CrossRef records found w/out affiliation filter, totalResults: ${results.totalResults} offset: ${offset}`) + if (results.totalResults <= recordLimit) { + console.log(`CrossRef getting records w/out affiliation filter, totalResults: ${results.totalResults} offset: ${offset}`) + totalResults = results.totalResults + publications = results.publications + itemsPerPage = results.itemsPerPage } else { - totalResults = 0 + // need to make sure date string in correct format + console.log(`CrossRef getting records w/ affiliation filter, offset: ${offset}`) + results = await this.fetchCrossRefResults(this.dsConfig.pageSize, offset, query['query.author'], query['query.affiliation'], query['filter']) + if (results.totalResults) { + totalResults = results.totalResults + publications = results.publications + itemsPerPage = results.itemsPerPage + } else { + totalResults = 0 + } } // console.log(`CrossRef results are: ${JSON.stringify(results['message']['items'][0], null, 2)}`) // console.log(`CrossRef results are: ${JSON.stringify(_.keys(results['message']), null, 2)}`) // console.log(`CrossRef results facets are: ${JSON.stringify(results['message']['facets'], null, 2)}`) - console.log(`CrossRef results total-results are: ${JSON.stringify(results['message']['total-results'], null, 2)}`) - console.log(`CrossRef results items-per-page are: ${JSON.stringify(results['message']['items-per-page'], null, 2)}`) + console.log(`CrossRef results total-results are: ${JSON.stringify(totalResults, null, 2)}`) + console.log(`CrossRef results items-per-page are: ${JSON.stringify(itemsPerPage, null, 2)}`) // console.log(`CrossRef results query is: ${JSON.stringify(results['message']['query'], null, 2)}`) @@ -83,6 +97,26 @@ export class CrossRefDataSource implements DataSource { return result } + async fetchCrossRefResults(pageSize, offset, queryAuthor, affiliation?, filter?) : Promise{ + // need to make sure date string in correct format + let totalResults + let publications + let itemsPerPage + const results = await this.fetchCrossRefQuery(this.dsConfig.pageSize, offset, queryAuthor, affiliation, filter) + if (results && results['message'] && results['message']['total-results']){ + totalResults = Number.parseInt(results['message']['total-results']) + itemsPerPage = results['message']['items-per-page'] + if (totalResults > 0 && results['message']['items']){ + publications = results['message']['items'] + } + } + return { + totalResults: totalResults, + publications: publications, + itemsPerPage: itemsPerPage + } + } + async fetchCrossRefQuery(pageSize, offset, queryAuthor, affiliation?, filter?) : Promise{ console.log(`Querying crossref offset: ${offset}, and query.author: ${queryAuthor} query.affiliation: ${affiliation} query.filter: ${filter}`) From 5355161745f4b2195f9a6fd114e6315c0eab5fbb Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 22 Jul 2021 13:12:07 -0400 Subject: [PATCH 14/43] remove ND from center review list, ui sync fixes --- client/src/pages/CenterReview.vue | 110 ++++++++----- client/src/pages/Index.vue | 145 ++++++++++-------- gql/readOrganizationsCenters.gql | 10 ++ .../down.yaml | 3 + .../up.yaml | 3 + .../down.yaml | 1 + .../up.yaml | 6 + .../down.yaml | 10 ++ .../up.yaml | 9 ++ .../down.yaml | 21 +++ .../up.yaml | 22 +++ .../down.yaml | 19 +++ .../up.yaml | 20 +++ .../down.yaml | 21 +++ .../up.yaml | 22 +++ hasura/migrations/metadata.yaml | 5 +- 16 files changed, 327 insertions(+), 100 deletions(-) create mode 100644 gql/readOrganizationsCenters.gql create mode 100644 hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml create mode 100644 hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml create mode 100644 hasura/migrations/1626972860860_add_review_organization_levels/down.yaml create mode 100644 hasura/migrations/1626972860860_add_review_organization_levels/up.yaml create mode 100644 hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml create mode 100644 hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml create mode 100644 hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index cdc88d44..11353210 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -186,6 +186,9 @@ target="_blank" /> + + Title: {{ publication.title }} + Citation: {{ publicationCitation }} @@ -416,7 +419,7 @@ import sanitize from 'sanitize-filename' import moment from 'moment' import pMap from 'p-map' import readPersonsByInstitutionByYearByOrganization from '../gql/readPersonsByInstitutionByYearByOrganization' -import readOrganizations from '../../../gql/readOrganizations.gql' +import readOrganizationsCenters from '../../../gql/readOrganizationsCenters.gql' // import VueFuse from 'vue-fuse' // Vue.use(VueFuse) @@ -429,6 +432,7 @@ export default { 'download-csv': JsonCSV }, data: () => ({ + pubLoadCount: 0, reviewStates: undefined, selectedReviewState: undefined, institutionReviewState: undefined, @@ -524,15 +528,19 @@ export default { watch: { $route: 'fetchData', selectedInstitutions: function () { + console.log('Selected Institutions Change. Reloading publications...') this.loadPublications() }, selectedCenter: function () { + console.log('Selected Center Change. Reloading publications...') this.loadPublications() }, changedPubYears: async function () { + console.log('Selected Pub Years Change. Reloading publications...') await this.loadPublications() }, changedMemberYears: async function () { + console.log('Selected Member Years Change. Reloading publications...') await this.loadPublications() }, selectedPersonSort: function () { @@ -560,10 +568,12 @@ export default { this.loadPersonsWithFilter() }, publicationsGroupedByView: function () { + console.log('Pubs Grouped By View Change. Reloading publications...') this.loadPublications() }, reviewTypeFilter: function () { if (this.publicationsReloadPending) { + console.log('Review Type Filter Change. Reloading publications...') this.loadPublications() this.publicationsReloadPending = false } else { @@ -1051,7 +1061,7 @@ export default { // }, async fetchData () { const results = await this.$apollo.query({ - query: readOrganizations + query: readOrganizationsCenters }) this.centerOptions = _.map(results.data.review_organization, (reviewOrg) => { @@ -1061,10 +1071,17 @@ export default { } }) + const centerValues = _.map(this.centerOptions, (option) => { return option.value }) + if (this.selectedCenter && this.selectedCenter.value && !_.includes(centerValues, this.selectedCenter.value)) { + // if a value not in list change to preferred + this.selectedCenter = undefined + } + if (!this.selectedCenter || !this.selectedCenter.value) { this.selectedCenter = this.preferredSelectedCenter } await this.loadReviewStates() + console.log('Fetching Data. Reloading publications...') await this.loadPublications() }, async clearPublications () { @@ -1461,6 +1478,10 @@ export default { } }, async loadPublications () { + const currentLoadCount = this.pubLoadCount + 1 + this.pubLoadCount += 1 + this.clearPublication() + this.clearPublications() this.startProgressBar() this.publicationsLoaded = false this.publicationsLoadedError = false @@ -1517,43 +1538,58 @@ export default { const personPubConfidenceSets = _.groupBy(pubsWithConfResult.data.confidencesets_persons_publications, (confPersonPub) => { return confPersonPub.persons_publications_id }) + let publicationIds + console.log(`Start query publications authors ${moment().format('HH:mm:ss:SSS')}`) - this.publications = _.map(pubsWithReviewResult.data.persons_publications, (personPub) => { - // change doi to lowercase - _.set(personPub.publication, 'doi', _.toLower(personPub.publication.doi)) - _.set(personPub, 'confidencesets', _.cloneDeep(personPubConfidenceSets[personPub.id])) - _.set(personPub, 'reviews', _.cloneDeep(personPubNDReviews[personPub.id])) - _.set(personPub, 'org_reviews', _.cloneDeep(personPubCenterReviews[personPub.id])) - return personPub - }) - const publicationIds = _.map(this.publications, (pub) => { - return pub.publication.id - }) - // now query for authors for the publications (faster if done in second query) - const pubsAuthorsResult = await this.$apollo.query({ - query: readAuthorsByPublications(publicationIds), - fetchPolicy: 'network-only' - }) - console.log(`Finished query publications authors ${moment().format('HH:mm:ss:SSS')}`) - const authorsPubs = _.map(pubsAuthorsResult.data.publications, (pub) => { - // change doi to lowercase - _.set(pub, 'doi', _.toLower(pub.doi)) - return pub - }) - const pubsWithAuthorsByDoi = _.groupBy(authorsPubs, (publication) => { - return publication.doi - }) - // now reduce to first instance by doi and authors array - this.authorsByDoi = _.mapValues(pubsWithAuthorsByDoi, (publication) => { - return (publication[0].authors) ? publication[0].authors : [] - }) - await this.loadPersonPublicationsCombinedMatches() - this.publicationsLoaded = true + if (currentLoadCount === this.pubLoadCount) { + this.publications = _.map(pubsWithReviewResult.data.persons_publications, (personPub) => { + // change doi to lowercase + _.set(personPub.publication, 'doi', _.toLower(personPub.publication.doi)) + _.set(personPub, 'confidencesets', _.cloneDeep(personPubConfidenceSets[personPub.id])) + _.set(personPub, 'reviews', _.cloneDeep(personPubNDReviews[personPub.id])) + _.set(personPub, 'org_reviews', _.cloneDeep(personPubCenterReviews[personPub.id])) + return personPub + }) + publicationIds = _.map(this.publications, (pub) => { + return pub.publication.id + }) + } - console.log(`Start query publications csl ${moment().format('HH:mm:ss:SSS')}`) - await this.loadPublicationsCSLData(publicationIds) - console.log(`Finished query publications csl ${moment().format('HH:mm:ss:SSS')}`) - this.publicationsCslLoaded = true + let pubsWithAuthorsByDoi + if (currentLoadCount === this.pubLoadCount) { + // now query for authors for the publications (faster if done in second query) + const pubsAuthorsResult = await this.$apollo.query({ + query: readAuthorsByPublications(publicationIds), + fetchPolicy: 'network-only' + }) + console.log(`Finished query publications authors ${moment().format('HH:mm:ss:SSS')}`) + const authorsPubs = _.map(pubsAuthorsResult.data.publications, (pub) => { + // change doi to lowercase + _.set(pub, 'doi', _.toLower(pub.doi)) + return pub + }) + pubsWithAuthorsByDoi = _.groupBy(authorsPubs, (publication) => { + return publication.doi + }) + } + + if (currentLoadCount === this.pubLoadCount) { + // now reduce to first instance by doi and authors array + this.authorsByDoi = _.mapValues(pubsWithAuthorsByDoi, (publication) => { + return (publication[0].authors) ? publication[0].authors : [] + }) + await this.loadPersonPublicationsCombinedMatches() + this.publicationsLoaded = true + } + + if (currentLoadCount === this.pubLoadCount) { + console.log(`Start query publications csl ${moment().format('HH:mm:ss:SSS')}`) + await this.loadPublicationsCSLData(publicationIds) + console.log(`Finished query publications csl ${moment().format('HH:mm:ss:SSS')}`) + this.publicationsCslLoaded = true + } else { + console.log('Reload of publications detected, aborting this process') + } } catch (error) { this.publicationsLoaded = true this.publicationsLoadedError = true diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 214138af..17a4b9e7 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -67,7 +67,7 @@ :active="person!==undefined && item.id === person.id" clickable group="expansion_group_person" - @click="resetReviewTypeFilter();startProgressBar();loadPublications(item); setNameVariants(item)" + @click="resetReviewTypeFilter();startProgressBar();clearPublication();clearPublications();loadPublications(item); setNameVariants(item)" active-class="bg-teal-1 text-grey-8" expand-icon="keyboard_arrow_rights" :ref="`person${index}`" @@ -230,6 +230,9 @@ target="_blank" /> + + Title: {{ publication.title }} + Citation: {{ publicationCitation }} @@ -429,6 +432,7 @@ export default { MainAuthorReviewFilter }, data: () => ({ + personLoadCount: 0, reviewStates: undefined, selectedReviewState: undefined, personScrollIndex: 0, @@ -841,6 +845,10 @@ export default { } }, async loadPersonsWithFilter () { + const currentLoadCount = this.personLoadCount + 1 + this.personLoadCount += 1 + this.clearPublication() + this.clearPublications() this.startPersonProgressBar() this.personsLoaded = false this.personsLoadedError = false @@ -851,77 +859,85 @@ export default { if (this.selectedPersonConfidence === '50%') minConfidence = 0.5 if (!this.selectedCenter || !this.selectedCenter.value || this.selectedCenter.value === 'ND') { const personResult = await this.$apollo.query(readPersonsByInstitutionByYearAllCenters(this.selectedInstitutions, this.selectedPubYears.min, this.selectedPubYears.max, this.selectedMemberYears.min, this.selectedMemberYears.max, minConfidence), { fetchPolicy: 'network-only' }) - this.people = personResult.data.persons + if (currentLoadCount === this.personLoadCount) { + this.people = personResult.data.persons + } } else { console.log(`Getting people for ${this.selectedCenter.value}`) const personResult = await this.$apollo.query(readPersonsByInstitutionByYearByOrganization(this.selectedCenter.value, this.selectedInstitutions, this.selectedPubYears.min, this.selectedPubYears.max, this.selectedMemberYears.min, this.selectedMemberYears.max, minConfidence), { fetchPolicy: 'network-only' }) - this.people = personResult.data.persons + if (currentLoadCount === this.personLoadCount) { + this.people = personResult.data.persons + } } - // calculate the total count to show - this.personReviewedPubCounts = {} - console.log('Checking for reviewed pub counts...') - _.each(this.people, (person) => { - const reviewedDois = {} - _.each(person.reviews_persons_publications, (review) => { - if (review.review_type && review.review_type !== 'pending') { - reviewedDois[review.doi] = review - } - }) - - // check for dois that are in the confidence set list and keep those, all others ignore - let filteredReviewedDoisCount = 0 - _.each(person.confidencesets_persons_publications, (confidenceSet) => { - const doi = confidenceSet.doi - if (reviewedDois[doi]) { - filteredReviewedDoisCount += 1 - } - }) + if (currentLoadCount === this.personLoadCount) { + // calculate the total count to show + this.personReviewedPubCounts = {} + console.log('Checking for reviewed pub counts...') + _.each(this.people, (person) => { + const reviewedDois = {} + _.each(person.reviews_persons_publications, (review) => { + if (review.review_type && review.review_type !== 'pending') { + reviewedDois[review.doi] = review + } + }) - this.personReviewedPubCounts[person.id] = filteredReviewedDoisCount - }) + // check for dois that are in the confidence set list and keep those, all others ignore + let filteredReviewedDoisCount = 0 + _.each(person.confidencesets_persons_publications, (confidenceSet) => { + const doi = confidenceSet.doi + if (reviewedDois[doi]) { + filteredReviewedDoisCount += 1 + } + }) - // console.log(`Reviewed counts are: ${JSON.stringify(this.personReviewedPubCounts, null, 2)}`) + this.personReviewedPubCounts[person.id] = filteredReviewedDoisCount + }) - // set the pub counts for person - this.people = _.map(this.people, (person) => { - return _.set(person, 'person_publication_count', this.getPersonPublicationCount(person)) - }) + // console.log(`Reviewed counts are: ${JSON.stringify(this.personReviewedPubCounts, null, 2)}`) - // apply any sorting applied - console.log('filtering', this.selectedPersonSort) - if (this.selectedPersonSort === 'Name') { - this.people = await _.sortBy(this.people, ['family_name', 'given_name']) - } else { - // need to sort by total and then name, not guaranteed to be in order from what is returned from DB - // first group items by count - const peopleByCounts = await _.groupBy(this.people, (person) => { - return this.getPersonPublicationCount(person) + // set the pub counts for person + this.people = _.map(this.people, (person) => { + return _.set(person, 'person_publication_count', this.getPersonPublicationCount(person)) }) - console.log(`People by counts are: ${JSON.stringify(peopleByCounts, null, 2)}`) + // apply any sorting applied + console.log('filtering', this.selectedPersonSort) + if (this.selectedPersonSort === 'Name') { + this.people = await _.sortBy(this.people, ['family_name', 'given_name']) + } else { + // need to sort by total and then name, not guaranteed to be in order from what is returned from DB + // first group items by count + const peopleByCounts = await _.groupBy(this.people, (person) => { + return this.getPersonPublicationCount(person) + }) - // sort each person array by name for each count - const peopleByCountsByName = await _.mapValues(peopleByCounts, (persons) => { - return _.sortBy(persons, ['family_name', 'given_name']) - }) + console.log(`People by counts are: ${JSON.stringify(peopleByCounts, null, 2)}`) - // get array of counts (i.e., keys) sorted in reverse - const sortedCounts = await _.sortBy(_.keys(peopleByCountsByName), (count) => { return Number.parseInt(count) }).reverse() + // sort each person array by name for each count + const peopleByCountsByName = await _.mapValues(peopleByCounts, (persons) => { + return _.sortBy(persons, ['family_name', 'given_name']) + }) - // now push values into array in desc order of count and flatten - let sortedPersons = [] - await _.each(sortedCounts, (key) => { - sortedPersons.push(peopleByCountsByName[key]) - }) + // get array of counts (i.e., keys) sorted in reverse + const sortedCounts = await _.sortBy(_.keys(peopleByCountsByName), (count) => { return Number.parseInt(count) }).reverse() - this.people = await _.flatten(sortedPersons) - // this.reportDuplicatePublications() - } - if (this.person) { - this.showCurrentSelectedPerson() + // now push values into array in desc order of count and flatten + let sortedPersons = [] + await _.each(sortedCounts, (key) => { + sortedPersons.push(peopleByCountsByName[key]) + }) + + this.people = await _.flatten(sortedPersons) + // this.reportDuplicatePublications() + } + if (this.person) { + this.showCurrentSelectedPerson() + } + this.personsLoaded = true + } else { + console.log('Another load of person detected before this process finished, aborting process.') } - this.personsLoaded = true }, async loadReviewStates () { console.log('loading review states') @@ -1212,12 +1228,17 @@ export default { }) // console.log('***', pubsWithReviewResult) console.log(`Finished query publications for person id: ${this.person.id} ${moment().format('HH:mm:ss:SSS')}`) - this.publications = _.map(pubsWithReviewResult.data.persons_publications, (personPub) => { - // change doi to lowercase - _.set(personPub.publication, 'doi', _.toLower(personPub.publication.doi)) - return personPub - }) - this.loadPersonPublicationsCombinedMatches() + // check if person selected changed when clicks happen rapidly and if so abort + if (this.person.id === person.id) { + this.publications = _.map(pubsWithReviewResult.data.persons_publications, (personPub) => { + // change doi to lowercase + _.set(personPub.publication, 'doi', _.toLower(personPub.publication.doi)) + return personPub + }) + this.loadPersonPublicationsCombinedMatches() + } else { + console.log(`Detected change in person selected abort query for person id: ${person.id}`) + } } catch (error) { this.publicationsLoaded = true this.publicationsLoadedError = true diff --git a/gql/readOrganizationsCenters.gql b/gql/readOrganizationsCenters.gql new file mode 100644 index 00000000..995b8141 --- /dev/null +++ b/gql/readOrganizationsCenters.gql @@ -0,0 +1,10 @@ +query MyQuery { + review_organization ( + where: { + level: {_gt: 1} + } + ){ + value + comment + } +} \ No newline at end of file diff --git a/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml b/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml new file mode 100644 index 00000000..31277656 --- /dev/null +++ b/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."review_organization" DROP COLUMN "level"; + type: run_sql diff --git a/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml b/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml new file mode 100644 index 00000000..27c90591 --- /dev/null +++ b/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."review_organization" ADD COLUMN "level" integer NULL; + type: run_sql diff --git a/hasura/migrations/1626972860860_add_review_organization_levels/down.yaml b/hasura/migrations/1626972860860_add_review_organization_levels/down.yaml new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/hasura/migrations/1626972860860_add_review_organization_levels/down.yaml @@ -0,0 +1 @@ +[] diff --git a/hasura/migrations/1626972860860_add_review_organization_levels/up.yaml b/hasura/migrations/1626972860860_add_review_organization_levels/up.yaml new file mode 100644 index 00000000..14f3f202 --- /dev/null +++ b/hasura/migrations/1626972860860_add_review_organization_levels/up.yaml @@ -0,0 +1,6 @@ +- args: + cascade: false + sql: |- + update review_organization set level = 1 where value = 'ND'; + update review_organization set level = 2 where value <> 'ND'; + type: run_sql diff --git a/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml b/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml new file mode 100644 index 00000000..7d5a4a41 --- /dev/null +++ b/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml @@ -0,0 +1,10 @@ +- args: + sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" TYPE integer; + type: run_sql +- args: + sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" DROP NOT + NULL; + type: run_sql +- args: + sql: COMMENT ON COLUMN "public"."review_organization"."level" IS E'null' + type: run_sql diff --git a/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml b/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml new file mode 100644 index 00000000..ee20f2f4 --- /dev/null +++ b/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml @@ -0,0 +1,9 @@ +- args: + sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" TYPE int4; + type: run_sql +- args: + sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" SET NOT NULL; + type: run_sql +- args: + sql: COMMENT ON COLUMN "public"."review_organization"."level" IS E'' + type: run_sql diff --git a/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml new file mode 100644 index 00000000..f2d64e7b --- /dev/null +++ b/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml @@ -0,0 +1,21 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_insert_permission +- args: + permission: + check: {} + columns: + - value + - comment + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml new file mode 100644 index 00000000..fc2aeea7 --- /dev/null +++ b/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml @@ -0,0 +1,22 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_insert_permission +- args: + permission: + check: {} + columns: + - comment + - level + - value + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml new file mode 100644 index 00000000..35faec2b --- /dev/null +++ b/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml @@ -0,0 +1,19 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_select_permission +- args: + permission: + allow_aggregations: true + columns: + - comment + - value + computed_fields: [] + filter: {} + role: user + table: + name: review_organization + schema: public + type: create_select_permission diff --git a/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml new file mode 100644 index 00000000..0c911b9b --- /dev/null +++ b/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml @@ -0,0 +1,20 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_select_permission +- args: + permission: + allow_aggregations: true + columns: + - comment + - level + - value + computed_fields: [] + filter: {} + role: user + table: + name: review_organization + schema: public + type: create_select_permission diff --git a/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml new file mode 100644 index 00000000..f7498bc6 --- /dev/null +++ b/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml @@ -0,0 +1,21 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_update_permission +- args: + permission: + columns: + - comment + - value + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization + schema: public + type: create_update_permission diff --git a/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml new file mode 100644 index 00000000..034174b6 --- /dev/null +++ b/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml @@ -0,0 +1,22 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_update_permission +- args: + permission: + columns: + - comment + - level + - value + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization + schema: public + type: create_update_permission diff --git a/hasura/migrations/metadata.yaml b/hasura/migrations/metadata.yaml index f6355b4e..1a6ee911 100644 --- a/hasura/migrations/metadata.yaml +++ b/hasura/migrations/metadata.yaml @@ -1537,8 +1537,9 @@ tables: set: {} check: {} columns: - - value - comment + - level + - value select_permissions: - role: user comment: null @@ -1547,6 +1548,7 @@ tables: computed_fields: [] columns: - comment + - level - value filter: {} update_permissions: @@ -1556,6 +1558,7 @@ tables: set: {} columns: - comment + - level - value filter: {} delete_permissions: [] From c9d93b85b202be3e26b5748d12b038d0908d0ba8 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 22 Jul 2021 14:19:00 -0400 Subject: [PATCH 15/43] fix for adding review_org level --- client/src/pages/CenterReview.vue | 17 +---- gql/readOrganizations.gql | 4 +- gql/readOrganizationsCenters.gql | 7 +- .../down.yaml | 7 ++ .../up.yaml | 3 + .../down.yaml | 6 ++ .../up.yaml | 6 ++ .../down.yaml | 6 ++ .../up.yaml | 8 +++ .../down.yaml | 6 ++ .../up.yaml | 12 ++++ .../down.yaml | 6 ++ .../up.yaml | 12 ++++ .../down.yaml | 6 ++ .../up.yaml | 16 +++++ .../down.yaml | 6 ++ .../up.yaml | 13 ++++ .../down.yaml | 6 ++ .../up.yaml | 15 +++++ .../down.yaml | 3 + .../up.yaml | 10 +++ .../down.yaml | 6 ++ .../up.yaml | 8 +++ .../down.yaml | 6 ++ .../up.yaml | 16 +++++ .../down.yaml | 6 ++ .../up.yaml | 13 ++++ .../down.yaml | 6 ++ .../up.yaml | 15 +++++ .../down.yaml | 1 + .../up.yaml | 9 +++ .../down.yaml | 6 ++ .../up.yaml | 12 ++++ .../down.yaml | 1 + .../up.yaml | 4 ++ .../down.yaml | 6 ++ .../up.yaml | 13 ++++ .../down.yaml | 12 ++++ .../up.yaml | 6 ++ .../down.yaml | 6 ++ .../up.yaml | 12 ++++ hasura/migrations/metadata.yaml | 66 +++++++++++++++++-- 42 files changed, 382 insertions(+), 23 deletions(-) create mode 100644 hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml create mode 100644 hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml create mode 100644 hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml create mode 100644 hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml create mode 100644 hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml create mode 100644 hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml create mode 100644 hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml create mode 100644 hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml create mode 100644 hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml create mode 100644 hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml create mode 100644 hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626976443229_create_table_public_review_organization_level/down.yaml create mode 100644 hasura/migrations/1626976443229_create_table_public_review_organization_level/up.yaml create mode 100644 hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/down.yaml create mode 100644 hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/up.yaml create mode 100644 hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/down.yaml create mode 100644 hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/up.yaml create mode 100644 hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/down.yaml create mode 100644 hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/up.yaml create mode 100644 hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/down.yaml create mode 100644 hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/up.yaml create mode 100644 hasura/migrations/1626976684591_insert_organization_levels/down.yaml create mode 100644 hasura/migrations/1626976684591_insert_organization_levels/up.yaml create mode 100644 hasura/migrations/1626976712095_add_relationship__table_public_review_organization/down.yaml create mode 100644 hasura/migrations/1626976712095_add_relationship__table_public_review_organization/up.yaml create mode 100644 hasura/migrations/1626977113039_insert_missing_organization_level/down.yaml create mode 100644 hasura/migrations/1626977113039_insert_missing_organization_level/up.yaml create mode 100644 hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/down.yaml create mode 100644 hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/up.yaml create mode 100644 hasura/migrations/1626977715477_add_relationship__table_public_review_organization/down.yaml create mode 100644 hasura/migrations/1626977715477_add_relationship__table_public_review_organization/up.yaml diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index 11353210..65049385 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -131,21 +131,6 @@ - - - - -       None - - - - @@ -186,7 +171,7 @@ target="_blank" /> - + Title: {{ publication.title }} diff --git a/gql/readOrganizations.gql b/gql/readOrganizations.gql index cb74ca24..ff0e87c8 100644 --- a/gql/readOrganizations.gql +++ b/gql/readOrganizations.gql @@ -1,5 +1,7 @@ query MyQuery { - review_organization { + review_organization ( + order_by: {review_organization_level: {level: asc}, comment: asc} + ){ value comment } diff --git a/gql/readOrganizationsCenters.gql b/gql/readOrganizationsCenters.gql index 995b8141..6240d4cd 100644 --- a/gql/readOrganizationsCenters.gql +++ b/gql/readOrganizationsCenters.gql @@ -1,8 +1,11 @@ query MyQuery { review_organization ( where: { - level: {_gt: 1} - } + review_organization_levels: { + level: {_gt: 1} + } + }, + order_by: {comment: asc} ){ value comment diff --git a/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml b/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml new file mode 100644 index 00000000..e2531123 --- /dev/null +++ b/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml @@ -0,0 +1,7 @@ +- args: + sql: ALTER TABLE "public"."review_organization" ADD COLUMN "level" int4 + type: run_sql +- args: + sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" DROP NOT + NULL + type: run_sql diff --git a/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml b/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml new file mode 100644 index 00000000..2eb68d9d --- /dev/null +++ b/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."review_organization" DROP COLUMN "level" CASCADE + type: run_sql diff --git a/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml b/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml new file mode 100644 index 00000000..80da6b31 --- /dev/null +++ b/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml @@ -0,0 +1,6 @@ +- args: + is_enum: false + table: + name: review_organization + schema: public + type: set_table_is_enum diff --git a/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml b/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml new file mode 100644 index 00000000..9ba18089 --- /dev/null +++ b/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml @@ -0,0 +1,6 @@ +- args: + is_enum: true + table: + name: review_organization + schema: public + type: set_table_is_enum diff --git a/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml b/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml new file mode 100644 index 00000000..0c38f445 --- /dev/null +++ b/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: review_organization + table: + name: persons_organizations + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml b/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml new file mode 100644 index 00000000..1eaa6cf3 --- /dev/null +++ b/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml @@ -0,0 +1,8 @@ +- args: + name: review_organization + table: + name: persons_organizations + schema: public + using: + foreign_key_constraint_on: organization_value + type: create_object_relationship diff --git a/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml b/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml new file mode 100644 index 00000000..7a19b3e4 --- /dev/null +++ b/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: persons_organizations + table: + name: review_organization + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml b/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml new file mode 100644 index 00000000..350eb5c7 --- /dev/null +++ b/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml @@ -0,0 +1,12 @@ +- args: + name: persons_organizations + table: + name: review_organization + schema: public + using: + foreign_key_constraint_on: + column: organization_value + table: + name: persons_organizations + schema: public + type: create_array_relationship diff --git a/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml b/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml new file mode 100644 index 00000000..cbcdb4ed --- /dev/null +++ b/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: reviews + table: + name: review_organization + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml b/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml new file mode 100644 index 00000000..ebb7b990 --- /dev/null +++ b/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml @@ -0,0 +1,12 @@ +- args: + name: reviews + table: + name: review_organization + schema: public + using: + foreign_key_constraint_on: + column: review_organization_value + table: + name: reviews + schema: public + type: create_array_relationship diff --git a/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml new file mode 100644 index 00000000..3b3695a0 --- /dev/null +++ b/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_insert_permission diff --git a/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml new file mode 100644 index 00000000..f46512f0 --- /dev/null +++ b/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml @@ -0,0 +1,16 @@ +- args: + permission: + allow_upsert: true + check: {} + columns: + - comment + - value + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml new file mode 100644 index 00000000..6261d93b --- /dev/null +++ b/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_select_permission diff --git a/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml new file mode 100644 index 00000000..bdb3de5a --- /dev/null +++ b/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml @@ -0,0 +1,13 @@ +- args: + permission: + allow_aggregations: true + columns: + - comment + - value + filter: {} + limit: null + role: user + table: + name: review_organization + schema: public + type: create_select_permission diff --git a/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml new file mode 100644 index 00000000..9aada99e --- /dev/null +++ b/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + role: user + table: + name: review_organization + schema: public + type: drop_update_permission diff --git a/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml new file mode 100644 index 00000000..32843009 --- /dev/null +++ b/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml @@ -0,0 +1,15 @@ +- args: + permission: + columns: + - comment + - value + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization + schema: public + type: create_update_permission diff --git a/hasura/migrations/1626976443229_create_table_public_review_organization_level/down.yaml b/hasura/migrations/1626976443229_create_table_public_review_organization_level/down.yaml new file mode 100644 index 00000000..c49a808f --- /dev/null +++ b/hasura/migrations/1626976443229_create_table_public_review_organization_level/down.yaml @@ -0,0 +1,3 @@ +- args: + sql: DROP TABLE "public"."review_organization_level" + type: run_sql diff --git a/hasura/migrations/1626976443229_create_table_public_review_organization_level/up.yaml b/hasura/migrations/1626976443229_create_table_public_review_organization_level/up.yaml new file mode 100644 index 00000000..bbb40393 --- /dev/null +++ b/hasura/migrations/1626976443229_create_table_public_review_organization_level/up.yaml @@ -0,0 +1,10 @@ +- args: + sql: CREATE TABLE "public"."review_organization_level"("organization_value" text + NOT NULL, "level" integer NOT NULL, PRIMARY KEY ("organization_value") , FOREIGN + KEY ("organization_value") REFERENCES "public"."review_organization"("value") + ON UPDATE no action ON DELETE no action, UNIQUE ("organization_value")); + type: run_sql +- args: + name: review_organization_level + schema: public + type: add_existing_table_or_view diff --git a/hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/down.yaml b/hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/down.yaml new file mode 100644 index 00000000..991a3cfd --- /dev/null +++ b/hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: review_organization + table: + name: review_organization_level + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/up.yaml b/hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/up.yaml new file mode 100644 index 00000000..d8a48142 --- /dev/null +++ b/hasura/migrations/1626976458392_add_relationship__table_public_review_organization_level/up.yaml @@ -0,0 +1,8 @@ +- args: + name: review_organization + table: + name: review_organization_level + schema: public + using: + foreign_key_constraint_on: organization_value + type: create_object_relationship diff --git a/hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/down.yaml b/hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/down.yaml new file mode 100644 index 00000000..e947c372 --- /dev/null +++ b/hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/down.yaml @@ -0,0 +1,6 @@ +- args: + role: user + table: + name: review_organization_level + schema: public + type: drop_insert_permission diff --git a/hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/up.yaml b/hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/up.yaml new file mode 100644 index 00000000..bef5fb05 --- /dev/null +++ b/hasura/migrations/1626976474341_update_permission_user_public_table_review_organization_level/up.yaml @@ -0,0 +1,16 @@ +- args: + permission: + allow_upsert: true + check: {} + columns: + - organization_value + - level + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization_level + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/down.yaml b/hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/down.yaml new file mode 100644 index 00000000..9b091b53 --- /dev/null +++ b/hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/down.yaml @@ -0,0 +1,6 @@ +- args: + role: user + table: + name: review_organization_level + schema: public + type: drop_select_permission diff --git a/hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/up.yaml b/hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/up.yaml new file mode 100644 index 00000000..7fe586e9 --- /dev/null +++ b/hasura/migrations/1626976483857_update_permission_user_public_table_review_organization_level/up.yaml @@ -0,0 +1,13 @@ +- args: + permission: + allow_aggregations: true + columns: + - level + - organization_value + filter: {} + limit: null + role: user + table: + name: review_organization_level + schema: public + type: create_select_permission diff --git a/hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/down.yaml b/hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/down.yaml new file mode 100644 index 00000000..510fd187 --- /dev/null +++ b/hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/down.yaml @@ -0,0 +1,6 @@ +- args: + role: user + table: + name: review_organization_level + schema: public + type: drop_update_permission diff --git a/hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/up.yaml b/hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/up.yaml new file mode 100644 index 00000000..f22c95ac --- /dev/null +++ b/hasura/migrations/1626976497196_update_permission_user_public_table_review_organization_level/up.yaml @@ -0,0 +1,15 @@ +- args: + permission: + columns: + - level + - organization_value + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: review_organization_level + schema: public + type: create_update_permission diff --git a/hasura/migrations/1626976684591_insert_organization_levels/down.yaml b/hasura/migrations/1626976684591_insert_organization_levels/down.yaml new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/hasura/migrations/1626976684591_insert_organization_levels/down.yaml @@ -0,0 +1 @@ +[] diff --git a/hasura/migrations/1626976684591_insert_organization_levels/up.yaml b/hasura/migrations/1626976684591_insert_organization_levels/up.yaml new file mode 100644 index 00000000..34978f66 --- /dev/null +++ b/hasura/migrations/1626976684591_insert_organization_levels/up.yaml @@ -0,0 +1,9 @@ +- args: + cascade: false + sql: "insert into review_organization_level values ('ND', 1);\ninsert into review_organization_level + values ('NDENERGY', 2);\ninsert into review_organization_level values ('EIGH', + 2);\ninsert into review_organization_level values ('NDNANO', 2);\ninsert into + review_organization_level values ('IPH', 2);\ninsert into review_organization_level + values ('ECI', 2);\ninsert into review_organization_level values ('CRC', 2);\ninsert + into review_organization_level values ('NDIAS', 2);\n " + type: run_sql diff --git a/hasura/migrations/1626976712095_add_relationship__table_public_review_organization/down.yaml b/hasura/migrations/1626976712095_add_relationship__table_public_review_organization/down.yaml new file mode 100644 index 00000000..76845681 --- /dev/null +++ b/hasura/migrations/1626976712095_add_relationship__table_public_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: review_organization_levels + table: + name: review_organization + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626976712095_add_relationship__table_public_review_organization/up.yaml b/hasura/migrations/1626976712095_add_relationship__table_public_review_organization/up.yaml new file mode 100644 index 00000000..81954715 --- /dev/null +++ b/hasura/migrations/1626976712095_add_relationship__table_public_review_organization/up.yaml @@ -0,0 +1,12 @@ +- args: + name: review_organization_levels + table: + name: review_organization + schema: public + using: + foreign_key_constraint_on: + column: organization_value + table: + name: review_organization_level + schema: public + type: create_array_relationship diff --git a/hasura/migrations/1626977113039_insert_missing_organization_level/down.yaml b/hasura/migrations/1626977113039_insert_missing_organization_level/down.yaml new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/hasura/migrations/1626977113039_insert_missing_organization_level/down.yaml @@ -0,0 +1 @@ +[] diff --git a/hasura/migrations/1626977113039_insert_missing_organization_level/up.yaml b/hasura/migrations/1626977113039_insert_missing_organization_level/up.yaml new file mode 100644 index 00000000..b3b36ea5 --- /dev/null +++ b/hasura/migrations/1626977113039_insert_missing_organization_level/up.yaml @@ -0,0 +1,4 @@ +- args: + cascade: false + sql: insert into review_organization_level values ('HCRI', 2); + type: run_sql diff --git a/hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/down.yaml b/hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/down.yaml new file mode 100644 index 00000000..e47b8bfc --- /dev/null +++ b/hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: review_organization_level + table: + name: review_organization + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/up.yaml b/hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/up.yaml new file mode 100644 index 00000000..e1ee89a9 --- /dev/null +++ b/hasura/migrations/1626977695704_create_relationship_review_organization_level_public_table_review_organization/up.yaml @@ -0,0 +1,13 @@ +- args: + name: review_organization_level + table: + name: review_organization + schema: public + using: + manual_configuration: + column_mapping: + value: organization_value + remote_table: + name: review_organization_level + schema: public + type: create_object_relationship diff --git a/hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/down.yaml b/hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/down.yaml new file mode 100644 index 00000000..81954715 --- /dev/null +++ b/hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/down.yaml @@ -0,0 +1,12 @@ +- args: + name: review_organization_levels + table: + name: review_organization + schema: public + using: + foreign_key_constraint_on: + column: organization_value + table: + name: review_organization_level + schema: public + type: create_array_relationship diff --git a/hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/up.yaml b/hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/up.yaml new file mode 100644 index 00000000..76845681 --- /dev/null +++ b/hasura/migrations/1626977703225_drop_relationship_review_organization_levels_public_table_review_organization/up.yaml @@ -0,0 +1,6 @@ +- args: + relationship: review_organization_levels + table: + name: review_organization + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626977715477_add_relationship__table_public_review_organization/down.yaml b/hasura/migrations/1626977715477_add_relationship__table_public_review_organization/down.yaml new file mode 100644 index 00000000..76845681 --- /dev/null +++ b/hasura/migrations/1626977715477_add_relationship__table_public_review_organization/down.yaml @@ -0,0 +1,6 @@ +- args: + relationship: review_organization_levels + table: + name: review_organization + schema: public + type: drop_relationship diff --git a/hasura/migrations/1626977715477_add_relationship__table_public_review_organization/up.yaml b/hasura/migrations/1626977715477_add_relationship__table_public_review_organization/up.yaml new file mode 100644 index 00000000..81954715 --- /dev/null +++ b/hasura/migrations/1626977715477_add_relationship__table_public_review_organization/up.yaml @@ -0,0 +1,12 @@ +- args: + name: review_organization_levels + table: + name: review_organization + schema: public + using: + foreign_key_constraint_on: + column: organization_value + table: + name: review_organization_level + schema: public + type: create_array_relationship diff --git a/hasura/migrations/metadata.yaml b/hasura/migrations/metadata.yaml index 1a6ee911..68cf7ccb 100644 --- a/hasura/migrations/metadata.yaml +++ b/hasura/migrations/metadata.yaml @@ -1516,7 +1516,14 @@ tables: update: null delete: null custom_column_names: {} - object_relationships: [] + object_relationships: + - using: + manual_configuration: + remote_table: review_organization_level + column_mapping: + value: organization_value + name: review_organization_level + comment: null array_relationships: - using: foreign_key_constraint_on: @@ -1524,6 +1531,12 @@ tables: table: persons_organizations name: persons_organizations comment: null + - using: + foreign_key_constraint_on: + column: organization_value + table: review_organization_level + name: review_organization_levels + comment: null - using: foreign_key_constraint_on: column: review_organization_value @@ -1538,7 +1551,6 @@ tables: check: {} columns: - comment - - level - value select_permissions: - role: user @@ -1548,7 +1560,6 @@ tables: computed_fields: [] columns: - comment - - level - value filter: {} update_permissions: @@ -1558,12 +1569,59 @@ tables: set: {} columns: - comment - - level - value filter: {} delete_permissions: [] event_triggers: [] computed_fields: [] +- table: review_organization_level + is_enum: false + configuration: + custom_root_fields: + select: null + select_by_pk: null + select_aggregate: null + insert: null + update: null + delete: null + custom_column_names: {} + object_relationships: + - using: + foreign_key_constraint_on: organization_value + name: review_organization + comment: null + array_relationships: [] + insert_permissions: + - role: user + comment: null + permission: + set: {} + check: {} + columns: + - organization_value + - level + select_permissions: + - role: user + comment: null + permission: + allow_aggregations: true + computed_fields: [] + columns: + - level + - organization_value + filter: {} + update_permissions: + - role: user + comment: null + permission: + set: {} + columns: + - level + - organization_value + filter: {} + delete_permissions: [] + event_triggers: [] + computed_fields: [] - table: reviews is_enum: false configuration: From 2a7b68f799583c4e21a26c0a73b0cfc1ee4e9019 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 22 Jul 2021 19:24:32 -0400 Subject: [PATCH 16/43] remove broken migrations --- .../down.yaml | 3 --- .../up.yaml | 3 --- .../down.yaml | 1 - .../up.yaml | 6 ----- .../down.yaml | 10 --------- .../up.yaml | 9 -------- .../down.yaml | 21 ------------------ .../up.yaml | 22 ------------------- .../down.yaml | 19 ---------------- .../up.yaml | 20 ----------------- .../down.yaml | 21 ------------------ .../up.yaml | 22 ------------------- .../down.yaml | 7 ------ .../up.yaml | 3 --- .../down.yaml | 6 ----- .../up.yaml | 6 ----- .../down.yaml | 6 ----- .../up.yaml | 8 ------- .../down.yaml | 6 ----- .../up.yaml | 12 ---------- .../down.yaml | 6 ----- .../up.yaml | 12 ---------- .../down.yaml | 6 ----- .../up.yaml | 16 -------------- .../down.yaml | 6 ----- .../up.yaml | 13 ----------- .../down.yaml | 6 ----- .../up.yaml | 15 ------------- 28 files changed, 291 deletions(-) delete mode 100644 hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml delete mode 100644 hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml delete mode 100644 hasura/migrations/1626972860860_add_review_organization_levels/down.yaml delete mode 100644 hasura/migrations/1626972860860_add_review_organization_levels/up.yaml delete mode 100644 hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml delete mode 100644 hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml delete mode 100644 hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml delete mode 100644 hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml delete mode 100644 hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml delete mode 100644 hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml delete mode 100644 hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml delete mode 100644 hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml delete mode 100644 hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml delete mode 100644 hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml delete mode 100644 hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml delete mode 100644 hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml delete mode 100644 hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml delete mode 100644 hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml delete mode 100644 hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml delete mode 100644 hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml delete mode 100644 hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml delete mode 100644 hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml delete mode 100644 hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml delete mode 100644 hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml delete mode 100644 hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml delete mode 100644 hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml delete mode 100644 hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml delete mode 100644 hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml diff --git a/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml b/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml deleted file mode 100644 index 31277656..00000000 --- a/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/down.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- args: - sql: ALTER TABLE "public"."review_organization" DROP COLUMN "level"; - type: run_sql diff --git a/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml b/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml deleted file mode 100644 index 27c90591..00000000 --- a/hasura/migrations/1626972511121_alter_table_public_review_organization_add_column_level/up.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- args: - sql: ALTER TABLE "public"."review_organization" ADD COLUMN "level" integer NULL; - type: run_sql diff --git a/hasura/migrations/1626972860860_add_review_organization_levels/down.yaml b/hasura/migrations/1626972860860_add_review_organization_levels/down.yaml deleted file mode 100644 index fe51488c..00000000 --- a/hasura/migrations/1626972860860_add_review_organization_levels/down.yaml +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/hasura/migrations/1626972860860_add_review_organization_levels/up.yaml b/hasura/migrations/1626972860860_add_review_organization_levels/up.yaml deleted file mode 100644 index 14f3f202..00000000 --- a/hasura/migrations/1626972860860_add_review_organization_levels/up.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - cascade: false - sql: |- - update review_organization set level = 1 where value = 'ND'; - update review_organization set level = 2 where value <> 'ND'; - type: run_sql diff --git a/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml b/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml deleted file mode 100644 index 7d5a4a41..00000000 --- a/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/down.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- args: - sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" TYPE integer; - type: run_sql -- args: - sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" DROP NOT - NULL; - type: run_sql -- args: - sql: COMMENT ON COLUMN "public"."review_organization"."level" IS E'null' - type: run_sql diff --git a/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml b/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml deleted file mode 100644 index ee20f2f4..00000000 --- a/hasura/migrations/1626972880465_alter_table_public_review_organization_alter_column_level/up.yaml +++ /dev/null @@ -1,9 +0,0 @@ -- args: - sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" TYPE int4; - type: run_sql -- args: - sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" SET NOT NULL; - type: run_sql -- args: - sql: COMMENT ON COLUMN "public"."review_organization"."level" IS E'' - type: run_sql diff --git a/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml deleted file mode 100644 index f2d64e7b..00000000 --- a/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/down.yaml +++ /dev/null @@ -1,21 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_insert_permission -- args: - permission: - check: {} - columns: - - value - - comment - localPresets: - - key: "" - value: "" - set: {} - role: user - table: - name: review_organization - schema: public - type: create_insert_permission diff --git a/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml deleted file mode 100644 index fc2aeea7..00000000 --- a/hasura/migrations/1626973200777_update_permission_user_public_table_review_organization/up.yaml +++ /dev/null @@ -1,22 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_insert_permission -- args: - permission: - check: {} - columns: - - comment - - level - - value - localPresets: - - key: "" - value: "" - set: {} - role: user - table: - name: review_organization - schema: public - type: create_insert_permission diff --git a/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml deleted file mode 100644 index 35faec2b..00000000 --- a/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/down.yaml +++ /dev/null @@ -1,19 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_select_permission -- args: - permission: - allow_aggregations: true - columns: - - comment - - value - computed_fields: [] - filter: {} - role: user - table: - name: review_organization - schema: public - type: create_select_permission diff --git a/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml deleted file mode 100644 index 0c911b9b..00000000 --- a/hasura/migrations/1626973214018_update_permission_user_public_table_review_organization/up.yaml +++ /dev/null @@ -1,20 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_select_permission -- args: - permission: - allow_aggregations: true - columns: - - comment - - level - - value - computed_fields: [] - filter: {} - role: user - table: - name: review_organization - schema: public - type: create_select_permission diff --git a/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml deleted file mode 100644 index f7498bc6..00000000 --- a/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/down.yaml +++ /dev/null @@ -1,21 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_update_permission -- args: - permission: - columns: - - comment - - value - filter: {} - localPresets: - - key: "" - value: "" - set: {} - role: user - table: - name: review_organization - schema: public - type: create_update_permission diff --git a/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml deleted file mode 100644 index 034174b6..00000000 --- a/hasura/migrations/1626973220488_update_permission_user_public_table_review_organization/up.yaml +++ /dev/null @@ -1,22 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_update_permission -- args: - permission: - columns: - - comment - - level - - value - filter: {} - localPresets: - - key: "" - value: "" - set: {} - role: user - table: - name: review_organization - schema: public - type: create_update_permission diff --git a/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml b/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml deleted file mode 100644 index e2531123..00000000 --- a/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/down.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- args: - sql: ALTER TABLE "public"."review_organization" ADD COLUMN "level" int4 - type: run_sql -- args: - sql: ALTER TABLE "public"."review_organization" ALTER COLUMN "level" DROP NOT - NULL - type: run_sql diff --git a/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml b/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml deleted file mode 100644 index 2eb68d9d..00000000 --- a/hasura/migrations/1626975705200_alter_table_public_review_organization_drop_column_level/up.yaml +++ /dev/null @@ -1,3 +0,0 @@ -- args: - sql: ALTER TABLE "public"."review_organization" DROP COLUMN "level" CASCADE - type: run_sql diff --git a/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml b/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml deleted file mode 100644 index 80da6b31..00000000 --- a/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - is_enum: false - table: - name: review_organization - schema: public - type: set_table_is_enum diff --git a/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml b/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml deleted file mode 100644 index 9ba18089..00000000 --- a/hasura/migrations/1626975780091_alter_table_public_review_organization_set_enum_true/up.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - is_enum: true - table: - name: review_organization - schema: public - type: set_table_is_enum diff --git a/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml b/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml deleted file mode 100644 index 0c38f445..00000000 --- a/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - relationship: review_organization - table: - name: persons_organizations - schema: public - type: drop_relationship diff --git a/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml b/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml deleted file mode 100644 index 1eaa6cf3..00000000 --- a/hasura/migrations/1626975867472_add_relationship__table_public_persons_organizations/up.yaml +++ /dev/null @@ -1,8 +0,0 @@ -- args: - name: review_organization - table: - name: persons_organizations - schema: public - using: - foreign_key_constraint_on: organization_value - type: create_object_relationship diff --git a/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml b/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml deleted file mode 100644 index 7a19b3e4..00000000 --- a/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - relationship: persons_organizations - table: - name: review_organization - schema: public - type: drop_relationship diff --git a/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml b/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml deleted file mode 100644 index 350eb5c7..00000000 --- a/hasura/migrations/1626975928474_add_relationship__table_public_review_organization/up.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- args: - name: persons_organizations - table: - name: review_organization - schema: public - using: - foreign_key_constraint_on: - column: organization_value - table: - name: persons_organizations - schema: public - type: create_array_relationship diff --git a/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml b/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml deleted file mode 100644 index cbcdb4ed..00000000 --- a/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - relationship: reviews - table: - name: review_organization - schema: public - type: drop_relationship diff --git a/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml b/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml deleted file mode 100644 index ebb7b990..00000000 --- a/hasura/migrations/1626975932559_add_relationship__table_public_review_organization/up.yaml +++ /dev/null @@ -1,12 +0,0 @@ -- args: - name: reviews - table: - name: review_organization - schema: public - using: - foreign_key_constraint_on: - column: review_organization_value - table: - name: reviews - schema: public - type: create_array_relationship diff --git a/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml deleted file mode 100644 index 3b3695a0..00000000 --- a/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_insert_permission diff --git a/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml deleted file mode 100644 index f46512f0..00000000 --- a/hasura/migrations/1626976033625_update_permission_user_public_table_review_organization/up.yaml +++ /dev/null @@ -1,16 +0,0 @@ -- args: - permission: - allow_upsert: true - check: {} - columns: - - comment - - value - localPresets: - - key: "" - value: "" - set: {} - role: user - table: - name: review_organization - schema: public - type: create_insert_permission diff --git a/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml deleted file mode 100644 index 6261d93b..00000000 --- a/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_select_permission diff --git a/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml deleted file mode 100644 index bdb3de5a..00000000 --- a/hasura/migrations/1626976045745_update_permission_user_public_table_review_organization/up.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- args: - permission: - allow_aggregations: true - columns: - - comment - - value - filter: {} - limit: null - role: user - table: - name: review_organization - schema: public - type: create_select_permission diff --git a/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml b/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml deleted file mode 100644 index 9aada99e..00000000 --- a/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/down.yaml +++ /dev/null @@ -1,6 +0,0 @@ -- args: - role: user - table: - name: review_organization - schema: public - type: drop_update_permission diff --git a/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml b/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml deleted file mode 100644 index 32843009..00000000 --- a/hasura/migrations/1626976061168_update_permission_user_public_table_review_organization/up.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- args: - permission: - columns: - - comment - - value - filter: {} - localPresets: - - key: "" - value: "" - set: {} - role: user - table: - name: review_organization - schema: public - type: create_update_permission From 823467dcb8ae2a9ee55e090c5b9b3a6a5ea45510 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 23 Jul 2021 16:11:23 -0400 Subject: [PATCH 17/43] add metadata SemanticScholar for pubs w/out dois --- .../down.yaml | 3 + .../up.yaml | 3 + ingest/fetchSemanticScholarAuthorData.ts | 2 +- ingest/modules/bibTex.ts | 43 ++++++++++++ ingest/modules/dataSource.ts | 2 +- ingest/modules/harvester.ts | 7 +- ingest/modules/normedAuthor.ts | 10 +++ ingest/modules/normedPublication.ts | 68 ++++++++++++++++++- .../normedPublicationObjectToCSVMap.json | 8 ++- ingest/modules/semanticScholarDataSource.ts | 53 ++++++++++++--- 10 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/down.yaml create mode 100644 hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/up.yaml create mode 100644 ingest/modules/bibTex.ts create mode 100644 ingest/modules/normedAuthor.ts diff --git a/hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/down.yaml b/hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/down.yaml new file mode 100644 index 00000000..b5ef6f29 --- /dev/null +++ b/hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/down.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."publications" DROP COLUMN "source_id"; + type: run_sql diff --git a/hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/up.yaml b/hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/up.yaml new file mode 100644 index 00000000..8dfb3dc5 --- /dev/null +++ b/hasura/migrations/1626980747629_alter_table_public_publications_add_column_source_id/up.yaml @@ -0,0 +1,3 @@ +- args: + sql: ALTER TABLE "public"."publications" ADD COLUMN "source_id" text NULL; + type: run_sql diff --git a/ingest/fetchSemanticScholarAuthorData.ts b/ingest/fetchSemanticScholarAuthorData.ts index 905dc63c..40495b78 100644 --- a/ingest/fetchSemanticScholarAuthorData.ts +++ b/ingest/fetchSemanticScholarAuthorData.ts @@ -111,7 +111,7 @@ async function main (): Promise { let harvestPerson = _.clone(person) harvestPerson.sourceIds.semanticScholarIds = semanticScholarIds const harvestPersons = [harvestPerson] - await semanticScholarHarvester.harvestToCsv(resultsDir, persons, HarvestOperation.QUERY_BY_AUTHOR_ID, getDateObject(`${year}-01-01`), getDateObject(`${year}-12-31`), `${person.familyName}_${person.givenName}`) + await semanticScholarHarvester.harvestToCsv(resultsDir, harvestPersons, HarvestOperation.QUERY_BY_AUTHOR_ID, getDateObject(`${year}-01-01`), getDateObject(`${year}-12-31`), `${person.familyName}_${person.givenName}`) // await pMap(searchNames, async (searchName) => { await wait(1500) diff --git a/ingest/modules/bibTex.ts b/ingest/modules/bibTex.ts new file mode 100644 index 00000000..5f51e7cf --- /dev/null +++ b/ingest/modules/bibTex.ts @@ -0,0 +1,43 @@ +import _ from 'lodash' +import NormedAuthor from './normedAuthor' + +export default class BibTex { + title: string + journal: string + author: string + year: string + month?: string + day?: string + publisher?: string + url?: string + issn?: string + doi?: string + abstract?: string + number?: string + volume?: string + pages?: string + eprint?: string + + public static toString(bibTex: BibTex): string { + let bibStr = '@article{' + _.each(_.keys(bibTex), (key, index) => { + if (index > 0) { + bibStr = `${bibStr}, ` + } + bibStr = `${bibStr}${key} = {${bibTex[key]}}` + }) + bibStr = `${bibStr} }` + return bibStr + } + + public static getBibTexAuthors(normedAuthors: NormedAuthor[]): string { + let authorStr = '' + _.each (normedAuthors, (normedAuthor, index) => { + if (index !== 0){ + authorStr = `${authorStr} and ` + } + authorStr = `${authorStr}${normedAuthor.familyName}, ${normedAuthor.givenName}` + }) + return authorStr + } +} \ No newline at end of file diff --git a/ingest/modules/dataSource.ts b/ingest/modules/dataSource.ts index 6de1255b..b8410374 100644 --- a/ingest/modules/dataSource.ts +++ b/ingest/modules/dataSource.ts @@ -14,7 +14,7 @@ export default interface DataSource { getPublicationsByAuthorId?(person: NormedPerson, sessionState: {}, offset: Number, startDate?: Date, endDate?: Date) : Promise // returns an array of normalized publication objects given ones retrieved fron this datasource - getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[] + getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): Promise //returns a machine readable string version of this source getSourceName() : string diff --git a/ingest/modules/harvester.ts b/ingest/modules/harvester.ts index 66022c3e..520a613b 100644 --- a/ingest/modules/harvester.ts +++ b/ingest/modules/harvester.ts @@ -191,8 +191,11 @@ export class Harvester { } } console.log(`Querying ${this.ds.getSourceName()} with date: ${searchStartDate}, offset: ${offset}, found pubs: ${harvestSet.sourcePublications.length} person: ${person.familyName}, ${person.givenName}`) - const normedPublications: NormedPublication[] = this.ds.getNormedPublications(harvestSet.sourcePublications, person) + // console.log(`Source pubs are: ${harvestSet.sourcePublications.length}`) + const normedPublications: NormedPublication[] = await this.ds.getNormedPublications(harvestSet.sourcePublications, person) _.set(harvestSet, 'normedPublications',normedPublications) + // console.log(`Normed pubs are: ${harvestSet.normedPublications.length}`) + return harvestSet } @@ -236,6 +239,8 @@ export class Harvester { } }) try { + // console.log(`Harvestsets are: ${JSON.stringify(harvestSets[0].normedPublications[0], null, 2)}`) + // console.log(`Writing normed pubs to csv: ${JSON.stringify(normedPubs, null, 2)}`) await NormedPublication.writeToCSV(normedPubs, filePath) await NormedPublication.writeSourceMetadataToJSON(normedPubs, resultsFileDir) } catch (error) { diff --git a/ingest/modules/normedAuthor.ts b/ingest/modules/normedAuthor.ts new file mode 100644 index 00000000..39480669 --- /dev/null +++ b/ingest/modules/normedAuthor.ts @@ -0,0 +1,10 @@ +export default interface NormedAuthor { + familyName: string + givenNameInitial: string + givenName: string + affiliations: any[] + sourceIds: { + scopusAffiliationId?: string, + semanticScholarIds?: string[] + } +} \ No newline at end of file diff --git a/ingest/modules/normedPublication.ts b/ingest/modules/normedPublication.ts index 238a0579..58ce14bd 100644 --- a/ingest/modules/normedPublication.ts +++ b/ingest/modules/normedPublication.ts @@ -6,8 +6,10 @@ import { getDateObject } from '../units/dateRange' import { command as loadCsv } from '../units/loadCsv' import { command as writeCsv} from '../units/writeCsv' import NormedPerson from './normedPerson' +import NormedAuthor from './normedAuthor' import writeToJSONFile from '../units/writeToJSONFile' import { loadJSONFromFile } from '../units/loadJSONFromFile' +import BibTex from './bibTex' export default class NormedPublication { // ------ begin declare properties used when using NormedPublication like an interface @@ -17,12 +19,18 @@ export default class NormedPublication { abstract?: string title: string journalTitle: string + authors: NormedAuthor[] journalIssn?: string journalEIssn?: string doi: string publicationDate: string datasourceName: string sourceId?: string + sourceUrl?: string + publisher?: string + number?: string + volume?: string + pages?: string sourceMetadata?: Object // ------- end declare properties used when using NormedPublication like an interface @@ -147,6 +155,24 @@ export default class NormedPublication { if (pub.sourceId) { row[objectToCSVMap['sourceId']] = pub.sourceId } + if (pub.sourceUrl) { + row[objectToCSVMap['sourceUrl']] = pub.sourceUrl + } + if (pub.publisher) { + row[objectToCSVMap['publisher']] = pub.publisher + } + if (pub.number) { + row[objectToCSVMap['number']] = pub.number + } + if (pub.volume) { + row[objectToCSVMap['volume']] = pub.volume + } + if (pub.pages) { + row[objectToCSVMap['pages']] = pub.pages + } + if (pub.authors) { + row[objectToCSVMap['authors']] = JSON.stringify(pub.authors) + } // if (pub.sourceMetadata) { // // parse and get rid of any escaped quote characters // row[objectToCSVMap['sourceMetadata']] = JSON.stringify(pub.sourceMetadata) @@ -194,12 +220,12 @@ export default class NormedPublication { // assumes all column names in row passed in have been converted to lowercase const searchPersonFamilyNameColumn = objectToCSVMap['searchPerson']['familyName'] let pub: NormedPublication = { - title: (row[_.toLower(objectToCSVMap['title'])] ? row[_.toLower(objectToCSVMap['title'])] : row[_.keys(row)[0]]), journalTitle: row[_.toLower(objectToCSVMap['journalTitle'])], doi: row[_.toLower(objectToCSVMap['doi'])], publicationDate: row[_.toLower(objectToCSVMap['publicationDate'])], - datasourceName: row[_.toLower(objectToCSVMap['datasourceName'])] + datasourceName: row[_.toLower(objectToCSVMap['datasourceName'])], + authors: (row[_.toLower(objectToCSVMap['authors'])] ? JSON.parse(row[_.toLower(objectToCSVMap['authors'])]) : undefined) } // set optional properties, for search person first check if family name provided if (row[_.toLower(searchPersonFamilyNameColumn)]){ @@ -228,6 +254,21 @@ export default class NormedPublication { if (row[_.toLower(objectToCSVMap['sourceId'])]) { _.set(pub, 'sourceId', row[_.toLower(objectToCSVMap['sourceId'])]) } + if (row[_.toLower(objectToCSVMap['sourceUrl'])]) { + _.set(pub, 'sourceUrl', row[_.toLower(objectToCSVMap['sourceUrl'])]) + } + if (row[_.toLower(objectToCSVMap['publisher'])]) { + _.set(pub, 'publisher', row[_.toLower(objectToCSVMap['publisher'])]) + } + if (row[_.toLower(objectToCSVMap['number'])]) { + _.set(pub, 'number', row[_.toLower(objectToCSVMap['number'])]) + } + if (row[_.toLower(objectToCSVMap['volume'])]) { + _.set(pub, 'volume', row[_.toLower(objectToCSVMap['volume'])]) + } + if (row[_.toLower(objectToCSVMap['pages'])]) { + _.set(pub, 'pages', row[_.toLower(objectToCSVMap['pages'])]) + } if (row[_.toLower(objectToCSVMap['sourceMetadata'])]) { // parse and get rid of any escaped quote characters const sourceMetadata = JSON.parse(row[_.toLower(objectToCSVMap['sourceMetadata'])]) @@ -236,4 +277,27 @@ export default class NormedPublication { return pub } + + public static getBibTex (normedPub: NormedPublication): BibTex { + const date: Date = getDateObject(normedPub.publicationDate) + + let bib: BibTex = { + title: normedPub.title, + journal: normedPub.journalTitle, + year: `${date.getFullYear()}`, + author: BibTex.getBibTexAuthors(normedPub.authors), + } + + // optional properties + if (normedPub.publisher) bib.publisher = normedPub.publisher + if (normedPub.sourceUrl) bib.url = normedPub.sourceUrl + if (normedPub.journalIssn) bib.issn = normedPub.journalIssn + if (normedPub.doi) bib.doi = normedPub.doi + if (normedPub.abstract) bib.abstract = normedPub.abstract + if (normedPub.number) bib.number = normedPub.number + if (normedPub.volume) bib.volume = normedPub.volume + if (normedPub.pages) bib.pages = normedPub.pages + + return bib + } } \ No newline at end of file diff --git a/ingest/modules/normedPublicationObjectToCSVMap.json b/ingest/modules/normedPublicationObjectToCSVMap.json index 9f90b900..02f8ef4a 100644 --- a/ingest/modules/normedPublicationObjectToCSVMap.json +++ b/ingest/modules/normedPublicationObjectToCSVMap.json @@ -19,5 +19,11 @@ "sourceMetadata": "source_metadata", "journalIssn": "journal_issn", "journalEIssn": "journal_eissn", - "abstract": "abstract" + "abstract": "abstract", + "authors": "authors", + "sourceUrl": "source_url", + "publisher": "publisher", + "number": "number", + "volume": "volume", + "pages": "pages" } diff --git a/ingest/modules/semanticScholarDataSource.ts b/ingest/modules/semanticScholarDataSource.ts index f824951e..f91bd051 100644 --- a/ingest/modules/semanticScholarDataSource.ts +++ b/ingest/modules/semanticScholarDataSource.ts @@ -2,6 +2,7 @@ import axios from 'axios' import _, { isInteger } from 'lodash' import NormedPublication from './normedPublication' import NormedPerson from './normedPerson' +import NormedAuthor from './normedAuthor' import DataSource from './dataSource' import HarvestSet from './HarvestSet' import DataSourceConfig from './dataSourceConfig' @@ -161,8 +162,11 @@ export class SemanticScholarDataSource implements DataSource { // } // returns an array of normalized publication objects given ones retrieved fron this datasource - getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[]{ - const normedPubs = _.map(sourcePublications, (pub) => { + async getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): Promise{ + let normedPubs = [] + await pMap (sourcePublications, async (pub) => { + const authors = await this.getNormedAuthors(pub) + console.log(`Normed authors found: ${JSON.stringify(authors, null, 2)}`) let normedPub: NormedPublication = { title: pub['title'], doi: pub['doi'], @@ -170,11 +174,13 @@ export class SemanticScholarDataSource implements DataSource { publicationDate: `${pub['year']}`, // force to be string datasourceName: this.getSourceName(), sourceId: pub['paperId'], + authors: authors, sourceMetadata: pub } // add optional properties if (searchPerson) _.set(normedPub, 'searchPerson', searchPerson) if (pub['abstract']) _.set(normedPub, 'abstract', pub['abstract']) + if (pub['url']) normedPub.sourceUrl = pub['url'] // if (pub['issn-type']) { // _.each(pub['issn-type'], (issn) => { // if (issn['type'] && issn['value']) { @@ -187,8 +193,8 @@ export class SemanticScholarDataSource implements DataSource { // }) // } // // console.log(`Created normed pub: ${JSON.stringify(normedPub, null, 2)}`) - return normedPub - }) + normedPubs.push(normedPub) + }, { concurrency: 1}) return _.filter(normedPubs, (pub) => { return (pub !== undefined && pub !== null) @@ -222,19 +228,46 @@ export class SemanticScholarDataSource implements DataSource { } async getCSLStyleAuthorList(sourceMetadata) { + // const sourceAuthors = this.getCoauthors(sourceMetadata) + // const cslStyleAuthors = [] + // await pMap(sourceAuthors, async (sourceAuthor, index) => { + // let author = _.clone(sourceAuthor) + // const parsedName = await nameParser({ + // name: sourceAuthor['name'], + // reduceMethod: 'majority', + // }); + // author['given'] = parsedName.first + // author['family'] = parsedName.last + // cslStyleAuthors.push(author) + // }, { concurrency: 1 }) + const normedAuthors: NormedAuthor[] = await this.getNormedAuthors(sourceMetadata) + return _.map(normedAuthors, (author) => { + return { + family: author.familyName, + given: author.givenName + } + }) + } + + async getNormedAuthors(sourceMetadata): Promise { const sourceAuthors = this.getCoauthors(sourceMetadata) - const cslStyleAuthors = [] + const normedAuthors: NormedAuthor[] = [] await pMap(sourceAuthors, async (sourceAuthor, index) => { - let author = _.clone(sourceAuthor) const parsedName = await nameParser({ name: sourceAuthor['name'], reduceMethod: 'majority', }); - author['given'] = parsedName.first - author['family'] = parsedName.last - cslStyleAuthors.push(author) + console.log(`Parsed name found is: ${JSON.stringify(parsedName, null, 2)}`) + let author: NormedAuthor = { + familyName: parsedName['last'], + givenName: parsedName['first'], + givenNameInitial: (parsedName['first'] ? parsedName['first'][0] : ''), + affiliations: [], + sourceIds: { semanticScholarIds : [sourceAuthor['authorId']]} + } + normedAuthors.push(author) }, { concurrency: 1 }) - return cslStyleAuthors + return normedAuthors } // returns map of person id to author ids From 92417fc0f4985a23c4ccc135edf936343dc23374 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 23 Jul 2021 16:15:44 -0400 Subject: [PATCH 18/43] add get bibtex properties from WoS --- ingest/modules/wosDataSource.ts | 56 +++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/ingest/modules/wosDataSource.ts b/ingest/modules/wosDataSource.ts index 4cd75e24..e716cea7 100644 --- a/ingest/modules/wosDataSource.ts +++ b/ingest/modules/wosDataSource.ts @@ -2,6 +2,7 @@ import axios, { AxiosResponse } from 'axios' import _ from 'lodash' const xmlToJson = require('xml-js'); import NormedPublication from './normedPublication' +import NormedAuthor from './normedAuthor' import NormedPerson from './normedPerson' import HarvestSet from './HarvestSet' import DataSource from './dataSource' @@ -185,26 +186,36 @@ export class WosDataSource implements DataSource { */ getWoSMapLabelsToValues(properties) { let transformedProperties = {} - _.each(properties, (property) => { + console.log(`properties are: ${JSON.stringify(properties, null, 2)}`) + let parseProperties = properties + if (!_.isArray(properties)){ + parseProperties = [properties] + } + _.each(parseProperties, (property) => { transformedProperties[property['label']['_text']] = property['value'] }) return transformedProperties } getAuthors(sourceMetadata) { + console.log(`Getting authors from source metadata: ${JSON.stringify(sourceMetadata, null, 2)}`) let authors = [] - if (sourceMetadata && sourceMetadata['authors'] && sourceMetadata['authors'].length > 0 && sourceMetadata['authors'][0]['value']){ - _.each(sourceMetadata['authors'][0]['value'], (author) => { - authors.push({name: author['_text']}) - }) + if (sourceMetadata && sourceMetadata['authors']){ + const sourceAuthors = this.getWoSMapLabelsToValues(sourceMetadata['authors']) + if (sourceAuthors['Authors'] && sourceAuthors['Authors'].length > 0){ + _.each(sourceAuthors['Authors'], (author) => { + authors.push({name: author['_text']}) + }) + } } return authors } async getCSLStyleAuthorList(sourceMetadata) { const sourceAuthors = this.getAuthors(sourceMetadata) + console.log(`source authors are: ${JSON.stringify(sourceAuthors, null, 2)}`) const cslStyleAuthors = [] - await pMap(sourceAuthors, async (sourceAuthor, index) => { + await pMap (sourceAuthors, async (sourceAuthor, index) => { let author = {} const parsedName = await nameParser({ name: sourceAuthor['name'], @@ -217,9 +228,28 @@ export class WosDataSource implements DataSource { return cslStyleAuthors } + async getNormedAuthors(sourceMetadata): Promise { + const cslAuthors = await this.getCSLStyleAuthorList(sourceMetadata) + console.log(`CSL authors are: ${JSON.stringify(cslAuthors, null, 2)}`) + const normedAuthors: NormedAuthor[] = [] + _.each(cslAuthors, (sourceAuthor, index) => { + let author: NormedAuthor = { + familyName: sourceAuthor.family, + givenName: sourceAuthor.given, + givenNameInitial: sourceAuthor.given[0], + affiliations: sourceAuthor.affiliation, + sourceIds: { } + } + normedAuthors.push(author) + }) + console.log(`Normed authors are: ${JSON.stringify(normedAuthors, null, 2)}`) + return normedAuthors + } + // returns an array of normalized publication objects given ones retrieved fron this datasource - getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[]{ - return _.map(sourcePublications, (pub) => { + async getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): Promise{ + let normedPubs = [] + await pMap (sourcePublications, async (pub) => { const otherProps = this.getWoSMapLabelsToValues(pub['other']) const sourceProps = this.getWoSMapLabelsToValues(pub['source']) @@ -230,6 +260,7 @@ export class WosDataSource implements DataSource { datasourceName: this.dsConfig.sourceName, doi: otherProps && otherProps['Identifier.Doi'] && otherProps['Identifier.Doi']['_text'] ? otherProps['Identifier.Doi']['_text'] : '', sourceId: pub['uid'] && pub['uid']['_text'] ? `${Number.parseInt(_.replace(pub['uid']['_text'], 'WOS:', ''))}` : '', + authors: await this.getNormedAuthors(pub), sourceMetadata: pub } // add optional properties @@ -240,8 +271,13 @@ export class WosDataSource implements DataSource { // don't worry about abstract for now if (otherProps && otherProps['Identifier.Issn'] && otherProps['Identifier.Issn']['_text']) _.set(normedPub, 'journalIssn', otherProps['Identifier.Issn']['_text']) if (otherProps && otherProps['Identifier.Eissn'] && otherProps['Identifier.Eissn']['_text']) _.set(normedPub, 'journalEIssn', otherProps['Identifier.Eissn']['_text']) - return normedPub - }) + + if (sourceProps && sourceProps['Issue'] && sourceProps['Issue']['_text']) _.set(normedPub, 'number', sourceProps['Issue']['_text']) + if (sourceProps && sourceProps['Volume'] && sourceProps['Volume']['_text']) _.set(normedPub, 'volume', sourceProps['Volume']['_text']) + if (sourceProps && sourceProps['Pages'] && sourceProps['Pages']['_text']) _.set(normedPub, 'pages', sourceProps['Pages']['_text']) + normedPubs.push(normedPub) + }, { concurrency: 1 }) + return normedPubs } //returns a machine readable string version of this source From 05c865c1de554f0407ee6724d0c94a3c26b9bb53 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 26 Jul 2021 22:05:48 -0400 Subject: [PATCH 19/43] insert pubs without dois to DB --- ingest/gql/readPublicationsBySourceId.js | 40 ++++++++++++ ingest/ingestMetadataByDoi.ts | 72 ++++++++++++++++----- ingest/modules/bibTex.ts | 2 +- ingest/modules/crossrefDataSource.ts | 66 ++++++++++++++++++- ingest/modules/pubmedDataSource.ts | 22 ++++++- ingest/modules/scopusDataSource.ts | 3 +- ingest/modules/semanticScholarDataSource.ts | 2 +- 7 files changed, 183 insertions(+), 24 deletions(-) create mode 100644 ingest/gql/readPublicationsBySourceId.js diff --git a/ingest/gql/readPublicationsBySourceId.js b/ingest/gql/readPublicationsBySourceId.js new file mode 100644 index 00000000..7b8738a5 --- /dev/null +++ b/ingest/gql/readPublicationsBySourceId.js @@ -0,0 +1,40 @@ +import gql from 'graphql-tag' + +export default function readPublicationsBySourceId (sourceName, sourceId) { + return { + query: gql` + query MyQuery ($source_name: String!, $source_id: String!){ + publications ( + where: { + source_name: {_eq: $source_name}, + source_id: {_eq: $source_id} + } + ){ + id + title + doi + year + csl_string + csl + source_name + source_metadata + source_id + scopus_eid: source_metadata(path: "eid") + scopus_pii: source_metadata(path: "pii") + wos_id: source_metadata(path: "uid") + pubmed_resource_identifiers: source_metadata(path: "resourceIdentifiers") + journal_title: csl(path: "container-title") + publications_authors { + id + given_name + family_name + } + } + } + `, + variables: { + source_name: sourceName, + source_id: sourceId + } + } +} diff --git a/ingest/ingestMetadataByDoi.ts b/ingest/ingestMetadataByDoi.ts index 04168128..86ed2cd8 100644 --- a/ingest/ingestMetadataByDoi.ts +++ b/ingest/ingestMetadataByDoi.ts @@ -19,6 +19,7 @@ import moment from 'moment' import dotenv from 'dotenv' import readPublicationsByDoi from './gql/readPublicationsByDoi' +import readPublicationsBySourceId from './gql/readPublicationsBySourceId' import readPublicationsByTitle from './gql/readPublicationsByTitle' import { getAllSimplifiedPersons } from './modules/queryNormalizedPeople' @@ -31,6 +32,7 @@ import DataSourceConfig from './modules/dataSourceConfig' import { Mutex } from './units/mutex' import NormedPublication from './modules/normedPublication' +import BibTex from './modules/bibTex' // import insertReview from '../client/src/gql/insertReview' dotenv.config({ @@ -127,7 +129,7 @@ function getPublicationYear (csl) : Number { } -async function insertPublicationAndAuthors (title, doi, csl, authors, sourceName, sourceMetadata, minPublicationYear?) { +async function insertPublicationAndAuthors (title, doi, csl, authors, sourceName, sourceId, sourceMetadata, minPublicationYear?) { //console.log(`trying to insert pub: ${JSON.stringify(title,null,2)}, ${JSON.stringify(doi,null,2)}`) try { const publicationYear = getPublicationYear (csl) @@ -142,6 +144,7 @@ async function insertPublicationAndAuthors (title, doi, csl, authors, sourceName year: publicationYear, csl: csl, // put these in as JSONB source_name: sourceName, + source_id: sourceId, source_metadata: sourceMetadata, // put these in as JSONB, csl_string: JSON.stringify(csl) } @@ -204,7 +207,7 @@ async function getPapersByDoi (csvPath: string, dataDirPath?: string) { counter += 1 //strip off 'doi:' if present if (!paper.doi || paper.doi.length <= 0){ - return `${prefix}_undefined_${counter}` + return `${paper.datasourceName}_${paper.sourceId}` } else { return paper.doi } @@ -351,17 +354,32 @@ async function matchPeopleToPaperAuthors(publicationCSL, simplifiedPersons, pers return matchedPersonMap } -async function isPublicationAlreadyInDB (doi, csl, sourceName) : Promise { - const queryResult = await client.query(readPublicationsByDoi(doi)) +async function isPublicationAlreadyInDB (doi, sourceId, csl, sourceName) : Promise { let foundPub = false const title = csl.title const publicationYear = getPublicationYear(csl) - const authors = await getCSLAuthors(csl) - _.each(queryResult.data.publications, (publication) => { - if (publication.doi === doi && _.toLower(publication.source_name) === _.toLower(sourceName)) { - foundPub = true + if (doi !== null){ + const queryResult = await client.query(readPublicationsByDoi(doi)) + // console.log(`Publications found for doi: ${doi}, ${queryResult.data.publications.length}`) + if (queryResult.data.publications && queryResult.data.publications.length > 0){ + _.each(queryResult.data.publications, (publication) => { + if (publication.doi && publication.doi !== null && _.toLower(publication.doi) === _.toLower(doi) && _.toLower(publication.source_name) === _.toLower(sourceName)) { + foundPub = true + } + }) } - }) + } + if (!foundPub) { + const querySourceIdResult = await client.query(readPublicationsBySourceId(sourceName, sourceId)) + // const authors = await getCSLAuthors(csl) + // console.log(`Publications found for sourceId: ${sourceId}, ${querySourceIdResult.data.publications.length}`) + _.each(querySourceIdResult.data.publications, (publication) => { + // console.log(`checking publication for source id: ${sourceId}, publication: ${JSON.stringify(publication, null, 2)}`) + if (_.toLower(publication.source_id) === _.toLower(sourceId) && _.toLower(publication.source_name) === _.toLower(sourceName)) { + foundPub = true + } + }) + } if (!foundPub) { const titleQueryResult = await client.query(readPublicationsByTitle(title)) _.each(titleQueryResult.data.publications, (publication) => { @@ -545,17 +563,30 @@ async function loadPersonPapersFromCSV (personMap, paperPath, minPublicationYear //console.log(`For DOI: ${doi}, Found CSL: ${JSON.stringify(cslRecords,null,2)}`) csl = cslRecords[0] } catch (error) { - if (bibTexByDoi[doi] && _.keys(bibTexByDoi[doi]).length > 0){ + const normedPub = papersByDoi[doi][0] + let bibTexStr = undefined + let normedBibTex = undefined + if (normedPub){ + normedBibTex = NormedPublication.getBibTex(normedPub) + if (normedBibTex) bibTexStr = BibTex.toString(normedBibTex) + } + if (!bibTexStr || bibTexByDoi[doi] && _.keys(bibTexByDoi[doi]).length > 0) { console.log(`Trying to get csl from bibtex for confirmed doi: ${doi}...`) // manually construct csl from metadata in confirmed list - const bibTex = bibTexByDoi[doi][_.keys(bibTexByDoi[doi])[0]] - if (bibTex) { - // console.log(`Trying to get csl from bibtex for confirmed doi: ${doi}, for bibtex found...`) - cslRecords = await Cite.inputAsync(bibTex) + bibTexStr = bibTexByDoi[doi][_.keys(bibTexByDoi[doi])[0]] + } + if (bibTexStr) { + // console.log(`Trying to get csl from bibtex str doi: ${doi}, for bibtex str ${bibTexStr} found...`) + try { + cslRecords = await Cite.inputAsync(bibTexStr) csl = cslRecords[0] // console.log(`CSL constructed: ${JSON.stringify(csl, null, 2)}`) + } catch (error) { + console.log(`Error on csl from bibtex: ${bibTexStr}`) + throw (error) } } else { + console.log(`Throwing the error for doi: ${doi}`) throw (error) } } @@ -564,6 +595,7 @@ async function loadPersonPapersFromCSV (personMap, paperPath, minPublicationYear let authors = [] if (csl) { + // console.log(`Getting csl authors for doi: ${doi}`) authors = await getCSLAuthors(csl) } // default to the confirmed author list if no author list in the csl record @@ -657,10 +689,16 @@ async function loadPersonPapersFromCSV (personMap, paperPath, minPublicationYear doiStatus.skippedDOIs.push(doi) } else { await mutex.dispatch( async () => { - const pubFound = await isPublicationAlreadyInDB(doi, csl, doiStatus.sourceName) + const sourceId = firstPaper.sourceId + // reset doi if it is a placeholder + let checkDoi = doi + if (_.toLower(doi) === _.toLower(`${doiStatus.sourceName}_${sourceId}`)){ + checkDoi = null + } + const pubFound = await isPublicationAlreadyInDB(checkDoi, sourceId, csl, doiStatus.sourceName) if (!pubFound) { // console.log(`Inserting Publication #${processedCount} of total ${count} DOI: ${doi} from source: ${doiStatus.sourceName}`) - const publicationId = await insertPublicationAndAuthors(csl.title, doi, csl, authors, doiStatus.sourceName, sourceMetadata) + const publicationId = await insertPublicationAndAuthors(csl.title, checkDoi, csl, authors, doiStatus.sourceName, sourceId, sourceMetadata) // console.log('Finished Running Insert and starting next thread') //console.log(`Inserted pub: ${JSON.stringify(publicationId,null,2)}`) @@ -744,7 +782,7 @@ async function loadPersonPapersFromCSV (personMap, paperPath, minPublicationYear paper = _.set(paper, 'error', errorMessage) failedRecords[doiStatus.sourceName].push(paper) } - }) + }) } // console.log(`DOIs Failed: ${JSON.stringify(doiStatus.failedDOIs,null,2)}`) // console.log(`Error Messages: ${JSON.stringify(doiStatus.errorMessages,null,2)}`) diff --git a/ingest/modules/bibTex.ts b/ingest/modules/bibTex.ts index 5f51e7cf..927ea538 100644 --- a/ingest/modules/bibTex.ts +++ b/ingest/modules/bibTex.ts @@ -19,7 +19,7 @@ export default class BibTex { eprint?: string public static toString(bibTex: BibTex): string { - let bibStr = '@article{' + let bibStr = `@article{${bibTex.doi},` _.each(_.keys(bibTex), (key, index) => { if (index > 0) { bibStr = `${bibStr}, ` diff --git a/ingest/modules/crossrefDataSource.ts b/ingest/modules/crossrefDataSource.ts index 5b11c7dc..bbd69693 100644 --- a/ingest/modules/crossrefDataSource.ts +++ b/ingest/modules/crossrefDataSource.ts @@ -2,6 +2,7 @@ import axios from 'axios' import _ from 'lodash' import NormedPublication from './normedPublication' import NormedPerson from './normedPerson' +import NormedAuthor from './normedAuthor' import DataSource from './dataSource' import HarvestSet from './HarvestSet' import DataSourceConfig from './dataSourceConfig' @@ -136,8 +137,63 @@ export class CrossRefDataSource implements DataSource { return response.data } + getCSLAuthors(paperCsl){ + + const authMap = { + firstAuthors : [], + otherAuthors : [] + } + + let authorCount = 0 + _.each(paperCsl.author, async (author) => { + // skip if family_name undefined + if (author.family != undefined){ + authorCount += 1 + + //if given name empty change to empty string instead of null, so that insert completes + if (author.given === undefined) author.given = '' + + if (_.lowerCase(author.sequence) === 'first' ) { + authMap.firstAuthors.push(author) + } else { + authMap.otherAuthors.push(author) + } + } + }) + + //add author positions + authMap.firstAuthors = _.forEach(authMap.firstAuthors, function (author, index){ + author.position = index + 1 + }) + + authMap.otherAuthors = _.forEach(authMap.otherAuthors, function (author, index){ + author.position = index + 1 + authMap.firstAuthors.length + }) + + //concat author arrays together + const authors = _.concat(authMap.firstAuthors, authMap.otherAuthors) + + return authors + } + + getNormedAuthors(csl) { + const cslAuthors = this.getCSLAuthors(csl) + const normedAuthors: NormedAuthor[] = [] + _.each(cslAuthors, (sourceAuthor, index) => { + let author: NormedAuthor = { + familyName: sourceAuthor.family, + givenName: sourceAuthor.given, + givenNameInitial: sourceAuthor.given[0], + affiliations: sourceAuthor.affiliation, + sourceIds: { semanticScholarIds : [sourceAuthor['authorId']]} + } + normedAuthors.push(author) + }) + return normedAuthors + } + // returns an array of normalized publication objects given ones retrieved fron this datasource - getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[]{ + async getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): Promise{ const normedPubs = _.map(sourcePublications, (pub) => { let publicationDate = '' if (pub['issued'] && pub['issued']['date-parts'] && pub['issued']['date-parts'][0] && pub['issued']['date-parts'][0][0]) { @@ -158,7 +214,13 @@ export class CrossRefDataSource implements DataSource { datasourceName: this.dsConfig.sourceName, doi: pub['DOI'] ? pub['DOI'] : '', sourceId: pub['DOI'] ? pub['DOI'] : '', - sourceMetadata: pub + authors: this.getNormedAuthors(pub), + sourceUrl: pub['URL'] ? pub['URL'] : '', + number: pub['issue'] ? pub['issue'] : '', + publisher: pub['publisher'] ? pub['publisher'] : '', + volume: pub['volume'] ? pub['volume'] : '', + sourceMetadata: pub, + } // console.log(`Setting search person for normed pubs: ${JSON.stringify(searchPerson, null, 2)}`) // add optional properties diff --git a/ingest/modules/pubmedDataSource.ts b/ingest/modules/pubmedDataSource.ts index f645207f..d8fc9ce2 100644 --- a/ingest/modules/pubmedDataSource.ts +++ b/ingest/modules/pubmedDataSource.ts @@ -3,6 +3,7 @@ import _ from 'lodash' const xmlToJson = require('xml-js'); import NormedPublication from './normedPublication' import NormedPerson from './normedPerson' +import NormedAuthor from './normedAuthor' import HarvestSet from './HarvestSet' import DataSource from './dataSource' import { getDateString, getDateObject } from '../units/dateRange' @@ -35,6 +36,19 @@ export class PubMedDataSource implements DataSource { return authors } + getNormedAuthors(sourceMetadata): NormedAuthor[] { + const authors = this.getAuthors(sourceMetadata) + return _.map(authors, (author) => { + return { + familyName: author.familyName, + givenName: author.givenName, + givenNameInitial: (author.givenName ? author.giveName[0]: ''), + affiliations: author.affiliations, + sourceIds: {} + } + }) + } + async getCSLStyleAuthorList(sourceMetadata) { const sourceAuthors = this.getAuthors(sourceMetadata) const cslStyleAuthors = [] @@ -61,7 +75,7 @@ export class PubMedDataSource implements DataSource { } // returns an array of normalized publication objects given ones retrieved fron this datasource - getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[]{ + async getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): Promise{ let normedPubs = [] const mappedOverObject = _.each(sourcePublications, (pub) => { const title = pub.title; @@ -89,15 +103,19 @@ export class PubMedDataSource implements DataSource { let pubmedId = identifiers.pubmed ? identifiers.pubmed.resourceIdentifier: '' console.log(`Creating normed pub for doi: ${doi} pubmed id: ${pubmedId}`) // update to be part of NormedPublication - let normedPub = { + let normedPub: NormedPublication = { title: title, journalTitle: '', doi: doi, publicationDate: pub.publicationYear, datasourceName: 'PubMed', sourceId: pubmedId, + authors: this.getNormedAuthors(pub), sourceMetadata: pub } + + if (searchPerson) _.set(normedPub, 'searchPerson', searchPerson) + if (pub['description']) _.set(normedPub, 'abstract', pub['description']) normedPubs.push(normedPub) }) return normedPubs diff --git a/ingest/modules/scopusDataSource.ts b/ingest/modules/scopusDataSource.ts index e34db4f8..5a06a406 100644 --- a/ingest/modules/scopusDataSource.ts +++ b/ingest/modules/scopusDataSource.ts @@ -79,7 +79,7 @@ export class ScopusDataSource implements DataSource { } // returns an array of normalized publication objects given ones retrieved fron this datasource - getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): NormedPublication[]{ + async getNormedPublications(sourcePublications: any[], searchPerson?: NormedPerson): Promise{ return _.map(sourcePublications, (pub) => { let normedPub: NormedPublication = { title: pub['dc:title'], @@ -88,6 +88,7 @@ export class ScopusDataSource implements DataSource { datasourceName: this.dsConfig.sourceName, doi: pub['prism:doi'] ? pub['prism:doi'] : '', sourceId: _.replace(pub['dc:identifier'], 'SCOPUS_ID:', ''), + authors: [], sourceMetadata: pub } // add optional properties diff --git a/ingest/modules/semanticScholarDataSource.ts b/ingest/modules/semanticScholarDataSource.ts index f91bd051..7f8a8fb0 100644 --- a/ingest/modules/semanticScholarDataSource.ts +++ b/ingest/modules/semanticScholarDataSource.ts @@ -257,7 +257,7 @@ export class SemanticScholarDataSource implements DataSource { name: sourceAuthor['name'], reduceMethod: 'majority', }); - console.log(`Parsed name found is: ${JSON.stringify(parsedName, null, 2)}`) + // console.log(`Parsed name found is: ${JSON.stringify(parsedName, null, 2)}`) let author: NormedAuthor = { familyName: parsedName['last'], givenName: parsedName['first'], From b6194e557c6316376fa76c152bcb0abc4895482a Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 26 Jul 2021 22:20:08 -0400 Subject: [PATCH 20/43] skip syncing review when pub has doi as null --- ingest/synchronizeReviewStates.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ingest/synchronizeReviewStates.ts b/ingest/synchronizeReviewStates.ts index 4b15ae96..bca2efbb 100644 --- a/ingest/synchronizeReviewStates.ts +++ b/ingest/synchronizeReviewStates.ts @@ -87,13 +87,15 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { _.each(reviewStates, (reviewType) => { const publications = publicationsGroupedByReview[reviewType] _.each(publications, (personPub) => { - if (!publicationDoisByReviewType[personPub['publication'].doi]) { - publicationDoisByReviewType[personPub['publication'].doi] = {} - } - if (!publicationDoisByReviewType[personPub['publication'].doi][reviewType]) { - publicationDoisByReviewType[personPub['publication'].doi][reviewType] = [] + if (personPub.doi !== null){ + if (!publicationDoisByReviewType[personPub['publication'].doi]) { + publicationDoisByReviewType[personPub['publication'].doi] = {} + } + if (!publicationDoisByReviewType[personPub['publication'].doi][reviewType]) { + publicationDoisByReviewType[personPub['publication'].doi][reviewType] = [] + } + publicationDoisByReviewType[personPub['publication'].doi][reviewType].push(personPub) } - publicationDoisByReviewType[personPub['publication'].doi][reviewType].push(personPub) }) }) From a08d52077624ea38e196e4e2c3d7e1443a05bd8f Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 27 Jul 2021 10:58:01 -0400 Subject: [PATCH 21/43] make author review group by title instead of doi --- ...eadPersonsByInstitutionByYearAllCenters.js | 16 +-- ...ersonsByInstitutionByYearByOrganization.js | 14 +-- client/src/gql/readPublications.js | 1 + client/src/pages/Index.vue | 99 ++++++++++--------- gql/readPersonPublications.gql | 1 + .../down.yaml | 31 ++++++ .../up.yaml | 32 ++++++ .../down.yaml | 29 ++++++ .../up.yaml | 30 ++++++ .../down.yaml | 31 ++++++ .../up.yaml | 32 ++++++ .../down.yaml | 1 + .../up.yaml | 11 +++ hasura/migrations/metadata.yaml | 3 + 14 files changed, 272 insertions(+), 59 deletions(-) create mode 100644 hasura/migrations/1627355129801_update_permission_user_public_table_publications/down.yaml create mode 100644 hasura/migrations/1627355129801_update_permission_user_public_table_publications/up.yaml create mode 100644 hasura/migrations/1627355138273_update_permission_user_public_table_publications/down.yaml create mode 100644 hasura/migrations/1627355138273_update_permission_user_public_table_publications/up.yaml create mode 100644 hasura/migrations/1627355145470_update_permission_user_public_table_publications/down.yaml create mode 100644 hasura/migrations/1627355145470_update_permission_user_public_table_publications/up.yaml create mode 100644 hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/down.yaml create mode 100644 hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/up.yaml diff --git a/client/src/gql/readPersonsByInstitutionByYearAllCenters.js b/client/src/gql/readPersonsByInstitutionByYearAllCenters.js index fee093af..56d1c11d 100644 --- a/client/src/gql/readPersonsByInstitutionByYearAllCenters.js +++ b/client/src/gql/readPersonsByInstitutionByYearAllCenters.js @@ -47,9 +47,9 @@ export default function readPersonsByInstitutionByYearAllCenters (institutionNam name } reviews_persons_publications( - distinct_on: doi, + distinct_on: title, order_by: { - doi: asc, + title: asc, datetime: desc }, where: { @@ -62,9 +62,9 @@ export default function readPersonsByInstitutionByYearAllCenters (institutionNam review_type } confidencesets_persons_publications( - distinct_on: doi, + distinct_on: title, order_by: { - doi: asc, + title: asc, datetime: desc }, where: { @@ -78,14 +78,14 @@ export default function readPersonsByInstitutionByYearAllCenters (institutionNam value year } - reviews_persons_publications_aggregate(distinct_on: doi, order_by: {doi: asc, datetime: desc}) { + reviews_persons_publications_aggregate(distinct_on: title, order_by: {title: asc, datetime: desc}) { aggregate { - count(columns: doi) + count(columns: title) } } - persons_publications_metadata_aggregate (distinct_on: doi, where: {year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}}}) { + persons_publications_metadata_aggregate (distinct_on: title, where: {year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}}}) { aggregate { - count(columns: doi) + count(columns: title) } } persons_namevariances { diff --git a/client/src/gql/readPersonsByInstitutionByYearByOrganization.js b/client/src/gql/readPersonsByInstitutionByYearByOrganization.js index dd7a703c..001aded5 100644 --- a/client/src/gql/readPersonsByInstitutionByYearByOrganization.js +++ b/client/src/gql/readPersonsByInstitutionByYearByOrganization.js @@ -49,9 +49,9 @@ export default function readPersonsByInstitutionByYearByOrganization (organizati name } confidencesets_persons_publications( - distinct_on: doi, + distinct_on: title order_by: { - doi: asc, + title: asc, datetime: desc }, where: { @@ -61,17 +61,17 @@ export default function readPersonsByInstitutionByYearByOrganization (organizati ) { doi value + title year } reviews_persons_publications( - distinct_on: doi, + distinct_on: title, order_by: { - doi: asc, + title: asc, datetime: desc }, where: { review_organization_value: {_eq: "ND"}, - review_type: {_neq: "pending"}, year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}} } ){ @@ -80,9 +80,9 @@ export default function readPersonsByInstitutionByYearByOrganization (organizati title review_type } - persons_publications_metadata_aggregate (distinct_on: doi, where: {year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}}}) { + persons_publications_metadata_aggregate (distinct_on: title, where: {year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}}}) { aggregate { - count(columns: doi) + count(columns: title) } } persons_namevariances { diff --git a/client/src/gql/readPublications.js b/client/src/gql/readPublications.js index 9bb57a60..97d6fe00 100644 --- a/client/src/gql/readPublications.js +++ b/client/src/gql/readPublications.js @@ -9,6 +9,7 @@ export default function readPublications () { title doi source_name + source_id } } ` diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 17a4b9e7..1b042ed8 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -114,10 +114,10 @@ v-model="reviewTypeFilter" dense > - - - - + + + +
- + View Article: { - const reviewedDois = {} + const reviewedTitles = {} _.each(person.reviews_persons_publications, (review) => { if (review.review_type && review.review_type !== 'pending') { - reviewedDois[review.doi] = review + reviewedTitles[review.title] = review } }) // check for dois that are in the confidence set list and keep those, all others ignore - let filteredReviewedDoisCount = 0 + let filteredReviewedTitlesCount = 0 _.each(person.confidencesets_persons_publications, (confidenceSet) => { - const doi = confidenceSet.doi - if (reviewedDois[doi]) { - filteredReviewedDoisCount += 1 + const title = confidenceSet.title + if (reviewedTitles[title]) { + filteredReviewedTitlesCount += 1 } }) - this.personReviewedPubCounts[person.id] = filteredReviewedDoisCount + this.personReviewedPubCounts[person.id] = filteredReviewedTitlesCount }) // console.log(`Reviewed counts are: ${JSON.stringify(this.personReviewedPubCounts, null, 2)}`) @@ -1010,8 +1017,8 @@ export default { this.personPublicationsCombinedMatches = [] this.personPublicationsCombinedMatchesByReview = {} this.filteredPersonPublicationsCombinedMatchesByReview = {} - this.publicationsGroupedByDoiByReview = {} - this.publicationsGroupedByDoi = {} + this.publicationsGroupedByTitleByReview = {} + this.publicationsGroupedByTitle = {} this.confidenceSetItems = [] this.confidenceSet = undefined }, @@ -1042,26 +1049,27 @@ export default { // check for any doi's with reviews out of sync, // if more than one review type found add doi mapped to array of reviewtype to array pub list - let publicationDoisByReviewType = {} + let publicationTitlesByReviewType = {} // put in pubs grouped by doi for each review status _.each(this.reviewStates, (reviewType) => { const publications = this.publicationsGroupedByReview[reviewType] - this.publicationsGroupedByDoiByReview[reviewType] = _.groupBy(publications, (personPub) => { - if (!publicationDoisByReviewType[personPub.publication.doi]) { - publicationDoisByReviewType[personPub.publication.doi] = {} + this.publicationsGroupedByTitleByReview[reviewType] = _.groupBy(publications, (personPub) => { + let title = personPub.publication.title + if (!publicationTitlesByReviewType[title]) { + publicationTitlesByReviewType[title] = {} } - if (!publicationDoisByReviewType[personPub.publication.doi][reviewType]) { - publicationDoisByReviewType[personPub.publication.doi][reviewType] = [] + if (!publicationTitlesByReviewType[title][reviewType]) { + publicationTitlesByReviewType[title][reviewType] = [] } - publicationDoisByReviewType[personPub.publication.doi][reviewType].push(personPub) - return `${personPub.publication.doi}` + publicationTitlesByReviewType[title][reviewType].push(personPub) + return `${title}` }) - // console.log(`Person pubs grouped by DOI are: ${JSON.stringify(this.publicationsGroupedByDoiByReview, null, 2)}`) - // grab one with highest confidence to display and grab others via doi later when changing status - this.personPublicationsCombinedMatchesByReview[reviewType] = _.map(_.keys(this.publicationsGroupedByDoiByReview[reviewType]), (doi) => { + // console.log(`Person pubs grouped by Title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) + // grab one with highest confidence to display and grab others via title later when changing status + this.personPublicationsCombinedMatchesByReview[reviewType] = _.map(_.keys(this.publicationsGroupedByTitleByReview[reviewType]), (title) => { // get match with highest confidence level and use that one - const personPubs = this.publicationsGroupedByDoiByReview[reviewType][doi] + const personPubs = this.publicationsGroupedByTitleByReview[reviewType][title] let currentPersonPub _.each(personPubs, (personPub, index) => { if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { @@ -1072,18 +1080,20 @@ export default { }) }) + // console.log(`Publications grouped by title by review: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) + // check for any doi's with reviews out of sync - const publicationDoisOutOfSync = [] + const publicationTitlesOutOfSync = [] - _.each(_.keys(publicationDoisByReviewType), (doi) => { - if (_.keys(publicationDoisByReviewType[doi]).length > 1) { - console.log(`Warning: Doi out of sync found: ${doi} for person id: ${this.person.id} doi record: ${JSON.stringify(publicationDoisByReviewType[doi], null, 2)}`) - publicationDoisOutOfSync.push(doi) + _.each(_.keys(publicationTitlesByReviewType), (title) => { + if (_.keys(publicationTitlesByReviewType[title]).length > 1) { + console.log(`Warning: Title out of sync found: ${title} for person id: ${this.person.id} title record: ${JSON.stringify(publicationTitlesByReviewType[title], null, 2)}`) + publicationTitlesOutOfSync.push(title) } }) - if (publicationDoisOutOfSync.length > 0) { - console.log(`Warning: Dois found with reviews out of sync: ${JSON.stringify(publicationDoisOutOfSync, null, 2)}`) + if (publicationTitlesOutOfSync.length > 0) { + console.log(`Warning: Titles found with reviews out of sync: ${JSON.stringify(publicationTitlesOutOfSync, null, 2)}`) } // initialize the list in view @@ -1249,8 +1259,8 @@ export default { async loadPublication (personPublication) { this.clearPublication() this.personPublication = personPublication - this.loadPublicationAuthors(personPublication) - this.loadConfidenceSet(personPublication) + await this.loadPublicationAuthors(personPublication) + await this.loadConfidenceSet(personPublication) // query separately for csl because slow to get more than one const publicationId = personPublication.publication.id const result = await this.$apollo.query({ @@ -1300,8 +1310,9 @@ export default { return null } this.person = person - // add the review for personPublications with the same doi in the list - const personPubs = this.publicationsGroupedByDoiByReview[this.reviewTypeFilter][personPublication.publication.doi] + // add the review for personPublications with the same title in the list + let title = personPublication.publication.title + const personPubs = this.publicationsGroupedByTitleByReview[this.reviewTypeFilter][title] try { let mutateResults = [] @@ -1317,7 +1328,7 @@ export default { Vue.delete(this.personPublicationsCombinedMatches, index) // transfer from one review queue to the next primarily for counts, other sorting will shake out on reload when clicking the tab // remove from current lists - _.unset(this.publicationsGroupedByDoiByReview[this.reviewTypeFilter], personPublication.publication.doi) + _.unset(this.publicationsGroupedByTitleByReview[this.reviewTypeFilter], title) _.remove(this.personPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (pub) => { return pub.id === personPub.id }) @@ -1325,7 +1336,7 @@ export default { return pub.id === personPub.id }) // add to new lists - this.publicationsGroupedByDoiByReview[reviewType][personPublication.publication.doi] = personPubs + this.publicationsGroupedByTitleByReview[reviewType][title] = personPubs this.personPublicationsCombinedMatchesByReview[reviewType].push(personPub) this.filteredPersonPublicationsCombinedMatchesByReview[reviewType].push(personPub) if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { diff --git a/gql/readPersonPublications.gql b/gql/readPersonPublications.gql index 7fbf5a97..8162ee13 100644 --- a/gql/readPersonPublications.gql +++ b/gql/readPersonPublications.gql @@ -16,6 +16,7 @@ query MyQuery ($personId: Int!, $yearMin: Int!, $yearMax: Int!) { title doi source_name + source_id pubmed_funders: source_metadata(path: "funderIdentifiers") crossref_funders: csl(path:"funder") scopus_eid: source_metadata(path: "eid") diff --git a/hasura/migrations/1627355129801_update_permission_user_public_table_publications/down.yaml b/hasura/migrations/1627355129801_update_permission_user_public_table_publications/down.yaml new file mode 100644 index 00000000..eb9342dc --- /dev/null +++ b/hasura/migrations/1627355129801_update_permission_user_public_table_publications/down.yaml @@ -0,0 +1,31 @@ +- args: + role: user + table: + name: publications + schema: public + type: drop_insert_permission +- args: + permission: + check: {} + columns: + - id + - title + - doi + - provenance_id + - data_id + - source_name + - csl + - source_metadata + - csl_string + - year + - abstract + - journal_id + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: publications + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1627355129801_update_permission_user_public_table_publications/up.yaml b/hasura/migrations/1627355129801_update_permission_user_public_table_publications/up.yaml new file mode 100644 index 00000000..9de4c5ad --- /dev/null +++ b/hasura/migrations/1627355129801_update_permission_user_public_table_publications/up.yaml @@ -0,0 +1,32 @@ +- args: + role: user + table: + name: publications + schema: public + type: drop_insert_permission +- args: + permission: + check: {} + columns: + - id + - title + - doi + - provenance_id + - data_id + - source_name + - csl + - source_metadata + - csl_string + - year + - abstract + - journal_id + - source_id + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: publications + schema: public + type: create_insert_permission diff --git a/hasura/migrations/1627355138273_update_permission_user_public_table_publications/down.yaml b/hasura/migrations/1627355138273_update_permission_user_public_table_publications/down.yaml new file mode 100644 index 00000000..b47b383b --- /dev/null +++ b/hasura/migrations/1627355138273_update_permission_user_public_table_publications/down.yaml @@ -0,0 +1,29 @@ +- args: + role: user + table: + name: publications + schema: public + type: drop_select_permission +- args: + permission: + allow_aggregations: true + columns: + - data_id + - id + - journal_id + - provenance_id + - year + - csl + - source_metadata + - abstract + - csl_string + - doi + - source_name + - title + computed_fields: [] + filter: {} + role: user + table: + name: publications + schema: public + type: create_select_permission diff --git a/hasura/migrations/1627355138273_update_permission_user_public_table_publications/up.yaml b/hasura/migrations/1627355138273_update_permission_user_public_table_publications/up.yaml new file mode 100644 index 00000000..e5acf22b --- /dev/null +++ b/hasura/migrations/1627355138273_update_permission_user_public_table_publications/up.yaml @@ -0,0 +1,30 @@ +- args: + role: user + table: + name: publications + schema: public + type: drop_select_permission +- args: + permission: + allow_aggregations: true + columns: + - data_id + - id + - journal_id + - provenance_id + - year + - csl + - source_metadata + - abstract + - csl_string + - doi + - source_id + - source_name + - title + computed_fields: [] + filter: {} + role: user + table: + name: publications + schema: public + type: create_select_permission diff --git a/hasura/migrations/1627355145470_update_permission_user_public_table_publications/down.yaml b/hasura/migrations/1627355145470_update_permission_user_public_table_publications/down.yaml new file mode 100644 index 00000000..58bb768c --- /dev/null +++ b/hasura/migrations/1627355145470_update_permission_user_public_table_publications/down.yaml @@ -0,0 +1,31 @@ +- args: + role: user + table: + name: publications + schema: public + type: drop_update_permission +- args: + permission: + columns: + - data_id + - id + - journal_id + - provenance_id + - year + - csl + - source_metadata + - abstract + - csl_string + - doi + - source_name + - title + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: publications + schema: public + type: create_update_permission diff --git a/hasura/migrations/1627355145470_update_permission_user_public_table_publications/up.yaml b/hasura/migrations/1627355145470_update_permission_user_public_table_publications/up.yaml new file mode 100644 index 00000000..f0136d12 --- /dev/null +++ b/hasura/migrations/1627355145470_update_permission_user_public_table_publications/up.yaml @@ -0,0 +1,32 @@ +- args: + role: user + table: + name: publications + schema: public + type: drop_update_permission +- args: + permission: + columns: + - data_id + - id + - journal_id + - provenance_id + - year + - csl + - source_metadata + - abstract + - csl_string + - doi + - source_id + - source_name + - title + filter: {} + localPresets: + - key: "" + value: "" + set: {} + role: user + table: + name: publications + schema: public + type: create_update_permission diff --git a/hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/down.yaml b/hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/down.yaml new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/down.yaml @@ -0,0 +1 @@ +[] diff --git a/hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/up.yaml b/hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/up.yaml new file mode 100644 index 00000000..8065b4ec --- /dev/null +++ b/hasura/migrations/1627389884638_add_source_id_confsets_persons_publications/up.yaml @@ -0,0 +1,11 @@ +- args: + cascade: false + sql: "CREATE OR REPLACE VIEW \"public\".\"confidencesets_persons_publications\" + AS \n SELECT confidencesets.id,\n confidencesets.persons_publications_id,\n + \ confidencesets.value,\n confidencesets.datetime,\n confidencesets.version,\n + \ persons_publications.person_id,\n persons_publications.publication_id,\n + \ publications.title,\n lower(publications.doi) AS doi,\n publications.source_name,\n + \ publications.year,\n publications.source_id\n FROM confidencesets,\n + \ persons_publications,\n publications\n WHERE ((confidencesets.persons_publications_id + = persons_publications.id) AND (persons_publications.publication_id = publications.id));" + type: run_sql diff --git a/hasura/migrations/metadata.yaml b/hasura/migrations/metadata.yaml index 68cf7ccb..f1015b27 100644 --- a/hasura/migrations/metadata.yaml +++ b/hasura/migrations/metadata.yaml @@ -1410,6 +1410,7 @@ tables: - year - abstract - journal_id + - source_id select_permissions: - role: user comment: null @@ -1427,6 +1428,7 @@ tables: - abstract - csl_string - doi + - source_id - source_name - title filter: {} @@ -1446,6 +1448,7 @@ tables: - abstract - csl_string - doi + - source_id - source_name - title filter: {} From 75919263da7db9511e29c3479f19b86432528eeb Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 27 Jul 2021 20:51:29 -0400 Subject: [PATCH 22/43] change synchronize reviews to sync on title --- ingest/synchronizeReviewStates.ts | 45 +++++++++++++++---------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/ingest/synchronizeReviewStates.ts b/ingest/synchronizeReviewStates.ts index bca2efbb..ab6c8c93 100644 --- a/ingest/synchronizeReviewStates.ts +++ b/ingest/synchronizeReviewStates.ts @@ -78,51 +78,50 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { }) // console.log(`Finish group by publications for person id: ${person.id}.`) - // check for any doi's with reviews out of sync, - // if more than one review type found add doi mapped to array of reviewtype to array pub list - let publicationDoisByReviewType = {} - let publicationsGroupedByDoiByReview = {} + // check for any title's with reviews out of sync, + // if more than one review type found add title mapped to array of reviewtype to array pub list + let publicationTitlesByReviewType = {} // put in pubs grouped by doi for each review status _.each(reviewStates, (reviewType) => { const publications = publicationsGroupedByReview[reviewType] _.each(publications, (personPub) => { if (personPub.doi !== null){ - if (!publicationDoisByReviewType[personPub['publication'].doi]) { - publicationDoisByReviewType[personPub['publication'].doi] = {} + if (!publicationTitlesByReviewType[personPub['publication'].title]) { + publicationTitlesByReviewType[personPub['publication'].title] = {} } - if (!publicationDoisByReviewType[personPub['publication'].doi][reviewType]) { - publicationDoisByReviewType[personPub['publication'].doi][reviewType] = [] + if (!publicationTitlesByReviewType[personPub['publication'].title][reviewType]) { + publicationTitlesByReviewType[personPub['publication'].title][reviewType] = [] } - publicationDoisByReviewType[personPub['publication'].doi][reviewType].push(personPub) + publicationTitlesByReviewType[personPub['publication'].title][reviewType].push(personPub) } }) }) - // check for any doi's with reviews out of sync - const publicationDoisOutOfSync = [] + // check for any title's with reviews out of sync + const publicationTitlesOutOfSync = [] - _.each(_.keys(publicationDoisByReviewType), (doi) => { - if (_.keys(publicationDoisByReviewType[doi]).length > 1) { + _.each(_.keys(publicationTitlesByReviewType), (title) => { + if (_.keys(publicationTitlesByReviewType[title]).length > 1) { // console.log(`Warning: Doi out of sync found: ${doi} for person id: ${this.person.id} doi record: ${JSON.stringify(publicationDoisByReviewType[doi], null, 2)}`) - publicationDoisOutOfSync.push(doi) + publicationTitlesOutOfSync.push(title) } }) - if (publicationDoisOutOfSync.length > 0) { - console.log(`Person id '${person['id']}' Dois found with reviews out of sync: ${JSON.stringify(publicationDoisOutOfSync, null, 2)}`) + if (publicationTitlesOutOfSync.length > 0) { + console.log(`Person id '${person['id']}' Dois found with reviews out of sync: ${JSON.stringify(publicationTitlesOutOfSync, null, 2)}`) console.log(`Synchronizing Reviews for these personPubs out of sync for person id '${person['id']}'...`) - await pMap(publicationDoisOutOfSync, async (doi) => { - // console.log(`Publication doi '${doi}' reviews by review type: ${JSON.stringify(publicationDoisByReviewType[doi], null, 2)}`) + await pMap(publicationTitlesOutOfSync, async (title) => { + // console.log(`Publication title '${title}' reviews by review type: ${JSON.stringify(publicationTitlesByReviewType[title], null, 2)}`) // get most recent review let mostRecentUpdateTime = undefined let newReview = {} let mostRecentReview = undefined let newReviewStatus = undefined let newReviewOrgValue = undefined - _.each(_.keys(publicationDoisByReviewType[doi]), (reviewType) => { + _.each(_.keys(publicationTitlesByReviewType[title]), (reviewType) => { // just get first one - const personPub = publicationDoisByReviewType[doi][reviewType][0] + const personPub = publicationTitlesByReviewType[title][reviewType][0] if (personPub['reviews_aggregate'].nodes.length > 0) { // console.log(`Looking at reviews for doi: ${doi}, reviewType: ${reviewType}, nodes: ${JSON.stringify(personPub['reviews_aggregate'].nodes, null, 2)}`) const currentDateTime = new Date(personPub['reviews_aggregate'].nodes[0].datetime) @@ -138,9 +137,9 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { }) if (mostRecentReview) { await pMap (reviewStates, async (reviewType) => { - if (reviewType !== mostRecentReview.reviewType && publicationDoisByReviewType[doi][reviewType]) { - await pMap (publicationDoisByReviewType[doi][reviewType], async (personPub) => { - console.log(`Inserting Review for doi: ${doi}, personpub: '${personPub['id']}', most recent review ${JSON.stringify(mostRecentReview, null, 2)}`) + if (reviewType !== mostRecentReview.reviewType && publicationTitlesByReviewType[title][reviewType]) { + await pMap (publicationTitlesByReviewType[title][reviewType], async (personPub) => { + console.log(`Inserting Review for title: ${title}, personpub: '${personPub['id']}', most recent review ${JSON.stringify(mostRecentReview, null, 2)}`) const mutateResult = await client.mutate( insertReview(mostRecentReview.userId, personPub['id'], mostRecentReview.reviewType, mostRecentReview.reviewOrgValue) ) From 18a33ec7e8a1bdfe228092a14c0b312793b6cab5 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Wed, 28 Jul 2021 19:48:44 -0400 Subject: [PATCH 23/43] center review group pubs by title instead of doi --- client/src/gql/readAuthorsByPublications.js | 1 + .../src/gql/readPersonPublicationsConfSets.js | 1 + .../src/gql/readPersonPublicationsReviews.js | 1 + client/src/gql/readPublicationsCSL.js | 1 + client/src/pages/CenterReview.vue | 190 +++++++++--------- 5 files changed, 99 insertions(+), 95 deletions(-) diff --git a/client/src/gql/readAuthorsByPublications.js b/client/src/gql/readAuthorsByPublications.js index 29f88c0e..4cac380c 100644 --- a/client/src/gql/readAuthorsByPublications.js +++ b/client/src/gql/readAuthorsByPublications.js @@ -20,6 +20,7 @@ export default function readAuthorsByPublications (publicationIds) { ){ id doi + title authors: csl(path: "author") } } diff --git a/client/src/gql/readPersonPublicationsConfSets.js b/client/src/gql/readPersonPublicationsConfSets.js index 88a127ee..760bb88c 100644 --- a/client/src/gql/readPersonPublicationsConfSets.js +++ b/client/src/gql/readPersonPublicationsConfSets.js @@ -19,6 +19,7 @@ export default function readPersonPublicationsConfSets (personPubIds) { person_id publication_id doi + title value datetime } diff --git a/client/src/gql/readPersonPublicationsReviews.js b/client/src/gql/readPersonPublicationsReviews.js index 5e87d6e2..307e3e7a 100644 --- a/client/src/gql/readPersonPublicationsReviews.js +++ b/client/src/gql/readPersonPublicationsReviews.js @@ -19,6 +19,7 @@ export default function readPersonPublicationsReviews (personPubIds, reviewOrgVa person_id publication_id doi + title review_type datetime } diff --git a/client/src/gql/readPublicationsCSL.js b/client/src/gql/readPublicationsCSL.js index 6e0bcd7d..5e83147a 100644 --- a/client/src/gql/readPublicationsCSL.js +++ b/client/src/gql/readPublicationsCSL.js @@ -20,6 +20,7 @@ export default function readPublicationsCSL (publicationIds) { ){ id doi + title csl_string } } diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index 65049385..aa761156 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -30,10 +30,10 @@ v-model="reviewTypeFilter" dense > - - - - + + + + Title: {{ decode(item.publication.title) }} - {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByDoi[selectedInstitutionReviewState.toLowerCase()][item.publication.doi] }} - {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByDoi[selectedInstitutionReviewState.toLowerCase()][item.publication.doi] }} + {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByTitle[selectedInstitutionReviewState.toLowerCase()][item.publication.title] }} + {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByTitle[selectedInstitutionReviewState.toLowerCase()][item.publication.title] }}
- + View Article: { const reviewedAuthor = this.getReviewedAuthor(personPub) @@ -689,11 +689,11 @@ export default { } return _.values(reviewedAuthors) }, - getDoiPersonPublicationsByReview (doi) { + getTitlePersonPublicationsByReview (title) { const personPubsByReview = {} - _.each(_.keys(this.publicationsGroupedByDoiByInstitutionReview), (reviewType) => { - if (this.publicationsGroupedByDoiByInstitutionReview[reviewType][doi]) { - const pubsGroupedByPersonId = _.groupBy(this.publicationsGroupedByDoiByInstitutionReview[reviewType][doi], (personPub) => { + _.each(_.keys(this.publicationsGroupedByTitleByInstitutionReview), (reviewType) => { + if (this.publicationsGroupedByTitleByInstitutionReview[reviewType][title]) { + const pubsGroupedByPersonId = _.groupBy(this.publicationsGroupedByTitleByInstitutionReview[reviewType][title], (personPub) => { return personPub.person_id }) personPubsByReview[reviewType] = _.map(_.keys(pubsGroupedByPersonId), (personId) => { @@ -1071,14 +1071,14 @@ export default { }, async clearPublications () { this.publications = [] - this.citationsByDoi = {} + this.citationsByTitle = {} this.personPublicationsCombinedMatches = [] this.personPublicationsCombinedMatchesByReview = {} this.personPublicationsCombinedMatchesByOrgReview = {} this.filteredPersonPublicationsCombinedMatchesByOrgReview = {} - this.publicationsGroupedByDoiByOrgReview = {} - this.publicationsGroupedByDoiByInstitutionReview = {} - this.publicationsGroupedByDoi = {} + this.publicationsGroupedByTitleByOrgReview = {} + this.publicationsGroupedByTitleByInstitutionReview = {} + this.publicationsGroupedByTitle = {} this.confidenceSetItems = [] this.confidenceSet = undefined this.filteredPersonPubCounts = {} @@ -1102,15 +1102,15 @@ export default { }) }, getPubCSVResultObject (personPublication) { - const citation = (this.citationsByDoi[_.toLower(personPublication.publication.doi)] ? this.citationsByDoi[_.toLower(personPublication.publication.doi)] : undefined) + const citation = (this.citationsByTitle[_.toLower(personPublication.publication.title)] ? this.citationsByTitle[_.toLower(personPublication.publication.title)] : undefined) return { - authors: this.sortAuthorsByDoi[this.selectedInstitutionReviewState.toLowerCase()][personPublication.publication.doi], + authors: this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][personPublication.publication.title], title: personPublication.publication.title.replace(/\n/g, ' '), doi: this.getCSVHyperLinkString(personPublication.publication.doi, this.getDoiUrl(personPublication.publication.doi)), journal: (personPublication.publication.journal_title) ? personPublication.publication.journal_title : '', year: personPublication.publication.year, - source_names: JSON.stringify(_.map(this.getSortedPersonPublicationsBySourceName(this.publicationsGroupedByDoi[personPublication.publication.doi]), (pub) => { return pub.publication.source_name })), - sources: this.getSourceUriString(this.getSortedPersonPublicationsBySourceName(this.publicationsGroupedByDoi[personPublication.publication.doi])), + source_names: JSON.stringify(_.map(this.getSortedPersonPublicationsBySourceName(this.publicationsGroupedByTitle[personPublication.publication.title]), (pub) => { return pub.publication.source_name })), + sources: this.getSourceUriString(this.getSortedPersonPublicationsBySourceName(this.publicationsGroupedByTitle[personPublication.publication.title])), abstract: personPublication.publication.abstract, citation: citation } @@ -1120,8 +1120,8 @@ export default { }, async loadPersonPublicationsCombinedMatches () { console.log(`Start group by publications ${moment().format('HH:mm:ss:SSS')}`) - this.publicationsGroupedByDoi = _.groupBy(this.publications, (personPub) => { - return `${personPub.publication.doi}` + this.publicationsGroupedByTitle = _.groupBy(this.publications, (personPub) => { + return `${personPub.publication.title}` }) // this.fundersByDoi = {} @@ -1130,58 +1130,58 @@ export default { // this.uniqueFunders = {} this.filteredPersonPubCounts = {} // group by institution (i.e., ND author) review and then by doi - let pubsByDoi = {} + let pubsByTitle = {} this.publicationsGroupedByInstitutionReview = _.groupBy(this.publications, function (personPub) { let reviewType = 'pending' - const doi = personPub.publication.doi + const title = personPub.publication.title // if (doi === '10.1101/gad.307116.117') { // console.log(`Person pub for doi 10.1101/gad.307116.117 is: ${JSON.stringify(personPub, null, 2)}`) // } if (personPub.reviews && personPub.reviews.length > 0) { reviewType = personPub.reviews[0].review_type } - if (!pubsByDoi[reviewType]) { - pubsByDoi[reviewType] = {} + if (!pubsByTitle[reviewType]) { + pubsByTitle[reviewType] = {} } - if (!pubsByDoi[reviewType][doi]) { - pubsByDoi[reviewType][doi] = [] + if (!pubsByTitle[reviewType][title]) { + pubsByTitle[reviewType][title] = [] } - pubsByDoi[reviewType][doi].push(personPub) + pubsByTitle[reviewType][title].push(personPub) return reviewType }) - // console.log(`Pubs by doi are: ${JSON.stringify(pubsByDoi, null, 2)}`) + // console.log(`Pubs by doi are: ${JSON.stringify(pubsByTitle, null, 2)}`) // put in pubs grouped by doi for each review status - this.publicationsGroupedByDoiByInstitutionReview = pubsByDoi + this.publicationsGroupedByTitleByInstitutionReview = pubsByTitle console.log(`Finish group by publications ${moment().format('HH:mm:ss:SSS')}`) // initialize the pub author matches - this.matchedPublicationAuthorsByDoi = _.mapValues(this.authorsByDoi, (cslAuthors, doi) => { - // console.log(`DOIs that are matched: ${JSON.stringify(_.keys(this.publicationsGroupedByDoiByInstitutionReview), null, 2)}`) - return this.getMatchedCslAuthors(cslAuthors, this.publicationsGroupedByDoiByInstitutionReview['accepted'][doi], true) + this.matchedPublicationAuthorsByTitle = _.mapValues(this.authorsByTitle, (cslAuthors, title) => { + // console.log(`DOIs that are matched: ${JSON.stringify(_.keys(this.publicationsGroupedByTitleByInstitutionReview), null, 2)}`) + return this.getMatchedCslAuthors(cslAuthors, this.publicationsGroupedByTitleByInstitutionReview['accepted'][title], true) }) - this.sortAuthorsByDoi = {} - this.sortAuthorsByDoi['accepted'] = _.mapValues(this.matchedPublicationAuthorsByDoi, (matchedAuthors) => { + this.sortAuthorsByTitle = {} + this.sortAuthorsByTitle['accepted'] = _.mapValues(this.matchedPublicationAuthorsByTitle, (matchedAuthors) => { this.updateFilteredPersonPubCounts('accepted', matchedAuthors) return this.getAuthorsString(matchedAuthors) }) - this.sortAuthorsByDoi['rejected'] = _.mapValues(this.publicationsGroupedByDoiByInstitutionReview['rejected'], (personPubs, doi) => { - return this.getAuthorsString(this.getInstitutionReviewedAuthors(doi, 'rejected')) + this.sortAuthorsByTitle['rejected'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['rejected'], (personPubs, title) => { + return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'rejected')) }) - this.sortAuthorsByDoi['unsure'] = _.mapValues(this.publicationsGroupedByDoiByInstitutionReview['unsure'], (personPubs, doi) => { - return this.getAuthorsString(this.getInstitutionReviewedAuthors(doi, 'unsure')) + this.sortAuthorsByTitle['unsure'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['unsure'], (personPubs, title) => { + return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'unsure')) }) - // console.log(`Matched pub authors by doi: ${JSON.stringify(this.matchedPublicationAuthorsByDoi, null, 2)}`) + // console.log(`Matched pub authors by doi: ${JSON.stringify(this.matchedPublicationAuthorsByTitle, null, 2)}`) - const doisPresent = {} + const titlesPresent = {} this.personPublicationsCombinedMatchesByReview = {} - // console.log(`Person pubs grouped by DOI are: ${JSON.stringify(this.publicationsGroupedByDoiByReview, null, 2)}`) - // grab one with highest confidence to display and grab others via doi later when changing status + // console.log(`Person pubs grouped by title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) + // grab one with highest confidence to display and grab others via title later when changing status // instead of random order add them sequentially to not have duplicates in rejected, // unsure if in accepted and then not in rejected if unsure and not in accepted - this.personPublicationsCombinedMatchesByReview['accepted'] = await _.map(_.keys(this.publicationsGroupedByDoiByInstitutionReview['accepted']), (doi) => { + this.personPublicationsCombinedMatchesByReview['accepted'] = await _.map(_.keys(this.publicationsGroupedByTitleByInstitutionReview['accepted']), (title) => { // get match with highest confidence level and use that one - const personPubs = this.publicationsGroupedByDoiByInstitutionReview['accepted'][doi] + const personPubs = this.publicationsGroupedByTitleByInstitutionReview['accepted'][title] let currentPersonPub _.each(personPubs, (personPub, index) => { if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { @@ -1227,15 +1227,15 @@ export default { // } } }) - doisPresent[doi] = true + titlesPresent[title] = true return currentPersonPub }) this.personPublicationsCombinedMatchesByReview['unsure'] = {} - await pMap(_.keys(this.publicationsGroupedByDoiByInstitutionReview['unsure']), (doi) => { + await pMap(_.keys(this.publicationsGroupedByTitleByInstitutionReview['unsure']), (title) => { // get match with highest confidence level and use that one - if (!doisPresent[doi]) { - const personPubs = this.publicationsGroupedByDoiByInstitutionReview['unsure'][doi] + if (!titlesPresent[title]) { + const personPubs = this.publicationsGroupedByTitleByInstitutionReview['unsure'][title] let currentPersonPub _.each(personPubs, (personPub, index) => { if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { @@ -1243,18 +1243,18 @@ export default { } }) - const unsureAuthors = this.getInstitutionReviewedAuthors(doi, 'unsure') + const unsureAuthors = this.getInstitutionReviewedAuthors(title, 'unsure') this.updateFilteredPersonPubCounts('unsure', unsureAuthors) - doisPresent[doi] = true - this.personPublicationsCombinedMatchesByReview['unsure'][doi] = currentPersonPub + titlesPresent[title] = true + this.personPublicationsCombinedMatchesByReview['unsure'][title] = currentPersonPub } }, { concurrency: 1 }) this.personPublicationsCombinedMatchesByReview['rejected'] = {} - await pMap(_.keys(this.publicationsGroupedByDoiByInstitutionReview['rejected']), (doi) => { + await pMap(_.keys(this.publicationsGroupedByTitleByInstitutionReview['rejected']), (title) => { // get match with highest confidence level and use that one - if (!doisPresent[doi]) { - const personPubs = this.publicationsGroupedByDoiByInstitutionReview['rejected'][doi] + if (!titlesPresent[title]) { + const personPubs = this.publicationsGroupedByTitleByInstitutionReview['rejected'][title] let currentPersonPub _.each(personPubs, (personPub, index) => { if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { @@ -1262,18 +1262,18 @@ export default { } }) - const rejectedAuthors = this.getInstitutionReviewedAuthors(doi, 'rejected') + const rejectedAuthors = this.getInstitutionReviewedAuthors(title, 'rejected') this.updateFilteredPersonPubCounts('rejected', rejectedAuthors) - doisPresent[doi] = true - this.personPublicationsCombinedMatchesByReview['rejected'][doi] = currentPersonPub + titlesPresent[title] = true + this.personPublicationsCombinedMatchesByReview['rejected'][title] = currentPersonPub } }, { concurrency: 1 }) this.personPublicationsCombinedMatchesByReview['pending'] = {} - await pMap(_.keys(this.publicationsGroupedByDoiByInstitutionReview['pending']), (doi) => { + await pMap(_.keys(this.publicationsGroupedByTitleByInstitutionReview['pending']), (title) => { // get match with highest confidence level and use that one - if (!doisPresent[doi]) { - const personPubs = this.publicationsGroupedByDoiByInstitutionReview['pending'][doi] + if (!titlesPresent[title]) { + const personPubs = this.publicationsGroupedByTitleByInstitutionReview['pending'][title] let currentPersonPub _.each(personPubs, (personPub, index) => { if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { @@ -1281,7 +1281,7 @@ export default { } }) - this.personPublicationsCombinedMatchesByReview['pending'][doi] = currentPersonPub + this.personPublicationsCombinedMatchesByReview['pending'][title] = currentPersonPub } }, { concurrency: 1 }) @@ -1304,12 +1304,12 @@ export default { } }) - // put in pubs grouped by doi for each review status for both ND author review and center review status + // put in pubs grouped by title for each review status for both ND author review and center review status _.each(this.reviewStates, (reviewType) => { const publications = this.personPublicationsCombinedMatchesByOrgReview[reviewType] - // first grab relevant person pubs from global list based dois in this list - this.publicationsGroupedByDoiByOrgReview[reviewType] = _.groupBy(publications, (personPub) => { - return `${personPub.publication.doi}` + // first grab relevant person pubs from global list based titles in this list + this.publicationsGroupedByTitleByOrgReview[reviewType] = _.groupBy(publications, (personPub) => { + return `${personPub.publication.title}` }) }) @@ -1347,7 +1347,7 @@ export default { this.personPublicationsCombinedMatchesByOrgReview, (personPublications) => { return _.filter(personPublications, (item) => { - const authorString = (this.sortAuthorsByDoi[this.selectedInstitutionReviewState.toLowerCase()][item.publication.doi]) ? this.sortAuthorsByDoi[this.selectedInstitutionReviewState.toLowerCase()][item.publication.doi] : '' + const authorString = (this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][item.publication.title]) ? this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][item.publication.title] : '' let includedInSelectedAuthors = true if (this.selectedCenterAuthor !== 'All') { // assumes the name value in the list of the same form as the author string @@ -1414,7 +1414,7 @@ export default { } else if (this.selectedCenterPubSort === 'Authors') { console.log('trying to sort by author') this.personPublicationsCombinedMatches = _.sortBy(this.personPublicationsCombinedMatches, (personPub) => { - return this.sortAuthorsByDoi[this.selectedInstitutionReviewState.toLowerCase()][personPub.publication.doi] + return this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][personPub.publication.title] }) } else if (this.selectedCenterPubSort === 'Source') { // need to sort by confidence and then name, not guaranteed to be in order from what is returned from DB @@ -1540,7 +1540,7 @@ export default { }) } - let pubsWithAuthorsByDoi + let pubsWithAuthorsByTitle if (currentLoadCount === this.pubLoadCount) { // now query for authors for the publications (faster if done in second query) const pubsAuthorsResult = await this.$apollo.query({ @@ -1553,14 +1553,14 @@ export default { _.set(pub, 'doi', _.toLower(pub.doi)) return pub }) - pubsWithAuthorsByDoi = _.groupBy(authorsPubs, (publication) => { - return publication.doi + pubsWithAuthorsByTitle = _.groupBy(authorsPubs, (publication) => { + return publication.title }) } if (currentLoadCount === this.pubLoadCount) { - // now reduce to first instance by doi and authors array - this.authorsByDoi = _.mapValues(pubsWithAuthorsByDoi, (publication) => { + // now reduce to first instance by title and authors array + this.authorsByTitle = _.mapValues(pubsWithAuthorsByTitle, (publication) => { return (publication[0].authors) ? publication[0].authors : [] }) await this.loadPersonPublicationsCombinedMatches() @@ -1588,11 +1588,11 @@ export default { query: readPublicationsCSL(publicationIds), fetchPolicy: 'network-only' }) - const publicationsCSLByDoi = _.groupBy(pubsCSLResult.data.publications, (publication) => { - return _.toLower(publication.doi) + const publicationsCSLByTitle = _.groupBy(pubsCSLResult.data.publications, (publication) => { + return _.toLower(publication.title) }) - this.citationsByDoi = _.mapValues(publicationsCSLByDoi, (publications) => { + this.citationsByTitle = _.mapValues(publicationsCSLByTitle, (publications) => { return this.getCitationApa(publications[0].csl_string) }) }, @@ -1618,7 +1618,7 @@ export default { async loadPublication (personPublication) { this.clearPublication() this.personPublication = personPublication - const personPublicationsByReview = await this.getDoiPersonPublicationsByReview(personPublication.publication.doi) + const personPublicationsByReview = await this.getTitlePersonPublicationsByReview(personPublication.publication.title) const reviewedAuthors = [] this.acceptedAuthors = _.map(personPublicationsByReview['accepted'], (personPub) => { const reviewedAuthor = this.getReviewedAuthor(personPub) @@ -1689,8 +1689,8 @@ export default { // TODO deselect buttons that are the same as the current filter return null } - // add the review for personPublications with the same doi in the list - const personPubs = this.publicationsGroupedByDoi[personPublication.publication.doi] + // add the review for personPublications with the same title in the list + const personPubs = this.publicationsGroupedByTitle[personPublication.publication.title] try { let mutateResults = [] @@ -1710,7 +1710,7 @@ export default { Vue.delete(this.personPublicationsCombinedMatches, index) // transfer from one review queue to the next primarily for counts, other sorting will shake out on reload when clicking the tab // remove from current lists - _.unset(this.publicationsGroupedByDoiByOrgReview[this.reviewTypeFilter], personPublication.publication.doi) + _.unset(this.publicationsGroupedByTitleByOrgReview[this.reviewTypeFilter], personPublication.publication.title) _.remove(this.personPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter], (pub) => { return pub.id === personPub.id }) @@ -1718,7 +1718,7 @@ export default { return pub.id === personPub.id }) // add to new lists - this.publicationsGroupedByDoiByOrgReview[reviewType][personPublication.publication.doi] = personPubs + this.publicationsGroupedByTitleByOrgReview[reviewType][personPublication.publication.title] = personPubs this.personPublicationsCombinedMatchesByOrgReview[reviewType].push(personPub) this.filteredPersonPublicationsCombinedMatchesByOrgReview[reviewType].push(personPub) if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { From 91c7e962c61a4964dad3c3ea94c489a535220a28 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 30 Jul 2021 16:46:25 -0400 Subject: [PATCH 24/43] accepted only in center screen, perf improvements --- .../src/components/CenterReviewPubFilter.vue | 5 +- client/src/pages/CenterReview.vue | 147 +++++++++++------- 2 files changed, 93 insertions(+), 59 deletions(-) diff --git a/client/src/components/CenterReviewPubFilter.vue b/client/src/components/CenterReviewPubFilter.vue index 1a5d8e5c..a399ef5f 100644 --- a/client/src/components/CenterReviewPubFilter.vue +++ b/client/src/components/CenterReviewPubFilter.vue @@ -18,7 +18,7 @@ - + />--> = 1) { - this.progress = 0.01 - this.buffer = 0.01 + this.progress = 0.95 + this.buffer = 0.95 return } - this.progress = Math.min(1, this.buffer, this.progress + 0.1) + let increment = 0.1 + if (this.progress > 0.6) increment = 0.01 + this.progress = Math.min(1, this.buffer, this.progress + increment) }, 700 + Math.random() * 1000) this.bufferInterval = setInterval(() => { @@ -848,32 +850,32 @@ export default { // apply any sorting applied console.log('filtering', this.selectedPersonSort) - if (this.selectedPersonSort === 'Name') { - this.people = await _.sortBy(this.people, ['family_name', 'given_name']) - } else { - // need to sort by total and then name, not guaranteed to be in order from what is returned from DB - // first group items by count - const peopleByCounts = await _.groupBy(this.people, (person) => { - return this.getFilteredPersonPubCount(this.selectedInstitutionReviewState.toLowerCase(), person) - }) - - // sort each person array by name for each count - const peopleByCountsByName = await _.mapValues(peopleByCounts, (persons) => { - return _.sortBy(persons, ['family_name', 'given_name']) - }) - - // get array of counts (i.e., keys) sorted in reverse - const sortedCounts = await _.sortBy(_.keys(peopleByCountsByName), (count) => { return Number.parseInt(count) }).reverse() - - // now push values into array in desc order of count and flatten - let sortedPersons = [] - await _.each(sortedCounts, (key) => { - sortedPersons.push(peopleByCountsByName[key]) - }) - - this.people = await _.flatten(sortedPersons) - // this.reportDuplicatePublications() - } + // if (this.selectedPersonSort === 'Name') { + this.people = await _.sortBy(this.people, ['family_name', 'given_name']) + // } else { + // // need to sort by total and then name, not guaranteed to be in order from what is returned from DB + // // first group items by count + // const peopleByCounts = await _.groupBy(this.people, (person) => { + // return this.getFilteredPersonPubCount(this.selectedInstitutionReviewState.toLowerCase(), person) + // }) + + // // sort each person array by name for each count + // const peopleByCountsByName = await _.mapValues(peopleByCounts, (persons) => { + // return _.sortBy(persons, ['family_name', 'given_name']) + // }) + + // // get array of counts (i.e., keys) sorted in reverse + // const sortedCounts = await _.sortBy(_.keys(peopleByCountsByName), (count) => { return Number.parseInt(count) }).reverse() + + // // now push values into array in desc order of count and flatten + // let sortedPersons = [] + // await _.each(sortedCounts, (key) => { + // sortedPersons.push(peopleByCountsByName[key]) + // }) + + // this.people = await _.flatten(sortedPersons) + // // this.reportDuplicatePublications() + // } await this.loadCenterAuthorOptions() }, async loadCenterAuthorOptions () { @@ -1072,6 +1074,9 @@ export default { async clearPublications () { this.publications = [] this.citationsByTitle = {} + this.people = [] + this.selectedCenterAuthor = 'All' + await this.loadCenterAuthorOptions() this.personPublicationsCombinedMatches = [] this.personPublicationsCombinedMatchesByReview = {} this.personPublicationsCombinedMatchesByOrgReview = {} @@ -1164,18 +1169,18 @@ export default { this.updateFilteredPersonPubCounts('accepted', matchedAuthors) return this.getAuthorsString(matchedAuthors) }) - this.sortAuthorsByTitle['rejected'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['rejected'], (personPubs, title) => { - return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'rejected')) - }) - this.sortAuthorsByTitle['unsure'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['unsure'], (personPubs, title) => { - return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'unsure')) - }) + // this.sortAuthorsByTitle['rejected'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['rejected'], (personPubs, title) => { + // return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'rejected')) + // }) + // this.sortAuthorsByTitle['unsure'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['unsure'], (personPubs, title) => { + // return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'unsure')) + // }) // console.log(`Matched pub authors by doi: ${JSON.stringify(this.matchedPublicationAuthorsByTitle, null, 2)}`) const titlesPresent = {} this.personPublicationsCombinedMatchesByReview = {} - // console.log(`Person pubs grouped by title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) + // console.log(`Person pubs grouped by title are: ${JSON.stringify(this.publicationsGroupedByTitleByInstitutionReview, null, 2)}`) // grab one with highest confidence to display and grab others via title later when changing status // instead of random order add them sequentially to not have duplicates in rejected, // unsure if in accepted and then not in rejected if unsure and not in accepted @@ -1495,10 +1500,15 @@ export default { }) // console.log('***', pubsWithReviewResult) console.log(`Finished query publications ND reviews ${moment().format('HH:mm:ss:SSS')}`) + const personPubNDReviewsByType = _.groupBy(pubsWithNDReviewsResult.data.reviews_persons_publications, (reviewPersonPub) => { + return reviewPersonPub.review_type + }) + const personPubNDReviews = _.groupBy(pubsWithNDReviewsResult.data.reviews_persons_publications, (reviewPersonPub) => { return reviewPersonPub.persons_publications_id }) + const personPubNDReviewsAccepted = personPubNDReviewsByType['accepted'] console.log(`This selectedCenter is: ${JSON.stringify(this.selectedCenter, null, 2)}`) console.log(`Starting query publications ${this.selectedCenter.value} reviews ${moment().format('HH:mm:ss:SSS')}`) @@ -1523,23 +1533,28 @@ export default { const personPubConfidenceSets = _.groupBy(pubsWithConfResult.data.confidencesets_persons_publications, (confPersonPub) => { return confPersonPub.persons_publications_id }) - let publicationIds + let singlePubIdsByTitle = {} console.log(`Start query publications authors ${moment().format('HH:mm:ss:SSS')}`) if (currentLoadCount === this.pubLoadCount) { - this.publications = _.map(pubsWithReviewResult.data.persons_publications, (personPub) => { + this.publications = _.map(personPubNDReviewsAccepted, (personPubReview) => { + const personPub = personPubByIds[personPubReview.persons_publications_id] + // const personPub = personPubNDReviewsAccepted[personPubId] + // grab the publication id and push to map to eliminate any dups + singlePubIdsByTitle[_.toLower(personPubReview.title)] = personPubReview.publication_id // change doi to lowercase - _.set(personPub.publication, 'doi', _.toLower(personPub.publication.doi)) - _.set(personPub, 'confidencesets', _.cloneDeep(personPubConfidenceSets[personPub.id])) - _.set(personPub, 'reviews', _.cloneDeep(personPubNDReviews[personPub.id])) - _.set(personPub, 'org_reviews', _.cloneDeep(personPubCenterReviews[personPub.id])) + _.set(personPub.publication, 'doi', _.toLower(personPubReview.doi)) + _.set(personPub, 'confidencesets', _.cloneDeep(personPubConfidenceSets[personPubReview.persons_publications_id])) + _.set(personPub, 'reviews', _.cloneDeep(personPubNDReviews[personPubReview.persons_publications_id])) + _.set(personPub, 'org_reviews', _.cloneDeep(personPubCenterReviews[personPubReview.persons_publications_id])) return personPub }) - publicationIds = _.map(this.publications, (pub) => { - return pub.publication.id - }) } + // console.log(`Publications found are: ${JSON.stringify(this.publications, null, 2)}`) + console.log(`On Query Total Titles Found: ${_.keys(singlePubIdsByTitle).length}`) + + const publicationIds = _.values(singlePubIdsByTitle) let pubsWithAuthorsByTitle if (currentLoadCount === this.pubLoadCount) { // now query for authors for the publications (faster if done in second query) @@ -1584,17 +1599,37 @@ export default { this.publicationsLoaded = true }, async loadPublicationsCSLData (publicationIds) { - const pubsCSLResult = await this.$apollo.query({ - query: readPublicationsCSL(publicationIds), - fetchPolicy: 'network-only' - }) - const publicationsCSLByTitle = _.groupBy(pubsCSLResult.data.publications, (publication) => { - return _.toLower(publication.title) - }) + this.citationsByTitle = {} + // break publicationIds into chunks of 50 + const batches = _.chunk(publicationIds, 2000) + let batchesPubsCSLByTitle = [] + console.log(`Getting csl for ${publicationIds.length} publication ids...`) + await pMap(batches, async (batch, index) => { + console.log(`Starting query csl for citations batch ${index} ${moment().format('HH:mm:ss:SSS')}`) + const pubsCSLResult = await this.$apollo.query({ + query: readPublicationsCSL(batch), + fetchPolicy: 'network-only' + }) + console.log(`Finished query csl for citations batch ${index} ${moment().format('HH:mm:ss:SSS')}`) - this.citationsByTitle = _.mapValues(publicationsCSLByTitle, (publications) => { - return this.getCitationApa(publications[0].csl_string) - }) + // console.log(`Starting group by title for citations batch ${index} ${moment().format('HH:mm:ss:SSS')}`) + batchesPubsCSLByTitle.push(_.groupBy(pubsCSLResult.data.publications, (publication) => { + return _.toLower(publication.title) + })) + // console.log(`Finished group by title for citations batch ${index} ${moment().format('HH:mm:ss:SSS')}`) + }, { concurrency: 1 }) + + // generate the citations themselves + console.log(`Starting generate citations ${moment().format('HH:mm:ss:SSS')}`) + await pMap(batchesPubsCSLByTitle, async (pubsCSLByTitle) => { + console.log(`Generating citations for ${_.keys(pubsCSLByTitle).length} publications`) + await pMap(_.keys(pubsCSLByTitle), async (title) => { + if (!this.citationsByTitle[title]) { + this.citationsByTitle[title] = this.getCitationApa(pubsCSLByTitle[title][0].csl_string) + } + }, { concurrency: 1 }) + }, { concurrency: 1 }) + console.log(`Finished generate citations ${moment().format('HH:mm:ss:SSS')}`) }, getReviewedAuthor (personPublication) { const obj = _.clone(personPublication.person) @@ -1858,7 +1893,7 @@ export default { this.selectedPersonSort = this.preferredPersonSort this.selectedPersonTotal = this.preferredPersonTotal this.selectedPersonConfidence = this.preferredPersonConfidence - this.selectedInstitutionReviewState = this.preferredInstitutionReviewState + this.selectedInstitutionReviewState = 'Accepted' // this.preferredInstitutionReviewState this.selectedPubYears = { min: this.yearPubStaticMin, max: this.yearPubStaticMax From 583a13283d363f35be04dc78b7c229b07fc2d5f7 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 2 Aug 2021 15:24:43 -0400 Subject: [PATCH 25/43] add conf set for any person pubs missing conf sets --- .../readPersonPublicationsConfSetsCount.js | 36 ++++++ ingest/modules/calculateConfidence.ts | 122 +++++++++--------- ingest/updateConfidenceReviewStates.ts | 17 +-- 3 files changed, 105 insertions(+), 70 deletions(-) create mode 100644 ingest/gql/readPersonPublicationsConfSetsCount.js diff --git a/ingest/gql/readPersonPublicationsConfSetsCount.js b/ingest/gql/readPersonPublicationsConfSetsCount.js new file mode 100644 index 00000000..4c91d64e --- /dev/null +++ b/ingest/gql/readPersonPublicationsConfSetsCount.js @@ -0,0 +1,36 @@ +import gql from 'graphql-tag' + +export default function readPersonPublicationsConfSetsCount (id, minConfidence) { + return { + query: gql` + query MyQuery ($person_id: Int!, $min_confidence: float8!){ + persons_publications ( + where: { + person_id: {_eq: $person_id}, + confidence: {_gte: $min_confidence} + }, + order_by: {confidencesets_aggregate: {count: asc}, id: asc} + ) { + id + person_id + confidence + publication { + id + doi + title + year + } + confidencesets_aggregate { + aggregate { + count(columns: persons_publications_id) + } + } + } + } + `, + variables: { + person_id: id, + min_confidence: minConfidence + } + } +} diff --git a/ingest/modules/calculateConfidence.ts b/ingest/modules/calculateConfidence.ts index 85362ddc..e03306a0 100644 --- a/ingest/modules/calculateConfidence.ts +++ b/ingest/modules/calculateConfidence.ts @@ -7,6 +7,7 @@ import fetch from 'node-fetch' import humanparser from 'humanparser' import readConfidenceTypes from '../gql/readConfidenceTypes' import readPersons from '../../client/src/gql/readPersons' +import readPersonPublicationsConfSetsCount from '../gql/readPersonPublicationsConfSetsCount' import readLastPersonPubConfidenceSet from '../gql/readLastPersonPubConfidenceSet' import readPersonPublications from '../gql/readPersonPublications' import readPersonPublication from '../gql/readPersonPublication' @@ -63,6 +64,19 @@ export class CalculateConfidence { constructor () { } + async getPersonPublicationsWithoutConfidenceSet (personId, minConfidence) { + const queryResult = await client.query(readPersonPublicationsConfSetsCount(personId, minConfidence)) + const personPubs = queryResult.data.persons_publications + const personPubByConfSetCount = _.groupBy(personPubs, (personPub) => { + return personPub.confidencesets_aggregate.aggregate.count + }) + if (personPubByConfSetCount['0']) { + return personPubByConfSetCount['0'] + } else { + return [] + } + } + // get the confidence set from the last run to know where to start calculating new confidence sets async getLastPersonPubConfidenceSet () { const queryResult = await client.query(readLastPersonPubConfidenceSet()) @@ -89,23 +103,22 @@ export class CalculateConfidence { } } - async getPersonPublications (personId, startPersonPubId, minConfidence, publicationYear?) { + async getPersonPublications (personId, minConfidence, overWriteExisting, publicationYear?) { console.log(`Getting Person Publications for person id: ${personId}`) if (publicationYear) { console.log(`Querying for person id: '${personId}' publications by year ${publicationYear}`) const queryResult = await client.query(readPersonPublicationsByYear(personId, publicationYear)) console.log(`Done querying for person id: '${personId}' publications by year ${publicationYear}`) return queryResult.data.persons_publications - } else if (startPersonPubId===undefined) { + } else if (overWriteExisting) { console.log(`Querying for person id: '${personId}' all publications`) const queryResult = await client.query(readPersonPublications(personId)) console.log(`Done querying for person id: '${personId}' all publications`) return queryResult.data.persons_publications } else { console.log(`Querying for person id: '${personId}' all recent publications`) - const queryResult = await client.query(readNewPersonPublications(personId, startPersonPubId, minConfidence)) + return this.getPersonPublicationsWithoutConfidenceSet(personId, minConfidence) console.log(`Done querying for person id: '${personId}' all recent publications`) - return queryResult.data.persons_publications } } @@ -724,7 +737,7 @@ testAuthorAffiliation (author, publicationAuthorMap, sourceName, sourceMetadata) // testAuthors: are authors for a given center/institute for the given year to test if there is a match // confirmedAuthors: is an optional parameter map of doi to a confirmed author if present and if so will make confidence highest // - async calculateConfidence (mostRecentPersonPubId, testAuthors, confirmedAuthors, publicationYear?) { + async calculateConfidence (testAuthors, confirmedAuthors, overWriteExisting, publicationYear?) { // get the set of tests to run const confidenceTypesByRank = await this.getConfidenceTypesByRank() @@ -738,63 +751,52 @@ testAuthorAffiliation (author, publicationAuthorMap, sourceName, sourceMetadata) await pMap(testAuthors, async (testAuthor) => { console.log(`Confidence Test Author is: ${testAuthor['names'][0]['lastName']}, ${testAuthor['names'][0]['firstName']}`) - // if most recent person pub id is defined, it will not recalculate past confidence sets - const personPubCount = await this.getPersonPublicationsCount(testAuthor['id'], mostRecentPersonPubId, minConfidence, publicationYear) - console.log(`Found '${personPubCount}' new possible pub matches for Test Author: ${testAuthor['names'][0]['lastName']}, ${testAuthor['names'][0]['firstName']}`) - - const resultLimit = 1000 - let numberOfRequests = 1 - if (personPubCount > resultLimit){ - numberOfRequests += parseInt(`${personPubCount / resultLimit}`) //convert to an integer to drop any decimal - } const thisConf = this - await pTimes (numberOfRequests, async function (index) { - console.log(`Performing request (${(index+1)} of ${numberOfRequests}) for Test Author: ${testAuthor['names'][0]['lastName']}, ${testAuthor['names'][0]['firstName']}`) - const personPublications = await thisConf.getPersonPublications(testAuthor['id'], mostRecentPersonPubId, minConfidence, publicationYear) - console.log(`Entering loop 2 Test Author: ${testAuthor['names'][0]['lastName']}`) - await pMap(personPublications, async (personPublication) => { - // need to load csl one by one since query fails otherwise - const currentPersonPublication = await thisConf.getPersonPublication(personPublication['id']) - const publicationCsl = JSON.parse(currentPersonPublication['publication']['csl_string']) - const sourceMetadata = currentPersonPublication['publication']['source_metadata'] - const sourceName = currentPersonPublication['publication']['source_name'] - // console.log(`Source metadata is: ${JSON.stringify(sourceMetadata, null, 2)}`) - const passedConfidenceTests = await thisConf.performAuthorConfidenceTests (testAuthor, publicationCsl, confirmedAuthors[personPublication['publication']['doi']], confidenceTypesByRank, sourceName, sourceMetadata) - - // returns a new map of rank -> confidenceTestName -> calculatedValue - const passedConfidenceTestsWithConf = await thisConf.calculateAuthorConfidence(passedConfidenceTests) - // calculate overall total and write the confidence set and comments to the DB - let confidenceTotal = 0.0 - _.mapValues(passedConfidenceTestsWithConf, (confidenceTests, rank) => { - _.mapValues(confidenceTests, (confidenceTest) => { - confidenceTotal += confidenceTest['confidenceValue'] - }) + const personPublications = await thisConf.getPersonPublications(testAuthor['id'], minConfidence, overWriteExisting, publicationYear) + console.log(`Found '${personPublications.length}' new possible pub matches for Test Author: ${testAuthor['names'][0]['lastName']}, ${testAuthor['names'][0]['firstName']}`) + console.log(`Entering loop 2 Test Author: ${testAuthor['names'][0]['lastName']}`) + await pMap(personPublications, async (personPublication) => { + // need to load csl one by one since query fails otherwise + const currentPersonPublication = await thisConf.getPersonPublication(personPublication['id']) + const publicationCsl = JSON.parse(currentPersonPublication['publication']['csl_string']) + const sourceMetadata = currentPersonPublication['publication']['source_metadata'] + const sourceName = currentPersonPublication['publication']['source_name'] + // console.log(`Source metadata is: ${JSON.stringify(sourceMetadata, null, 2)}`) + const passedConfidenceTests = await thisConf.performAuthorConfidenceTests (testAuthor, publicationCsl, confirmedAuthors[personPublication['publication']['doi']], confidenceTypesByRank, sourceName, sourceMetadata) + + // returns a new map of rank -> confidenceTestName -> calculatedValue + const passedConfidenceTestsWithConf = await thisConf.calculateAuthorConfidence(passedConfidenceTests) + // calculate overall total and write the confidence set and comments to the DB + let confidenceTotal = 0.0 + _.mapValues(passedConfidenceTestsWithConf, (confidenceTests, rank) => { + _.mapValues(confidenceTests, (confidenceTest) => { + confidenceTotal += confidenceTest['confidenceValue'] }) - // set ceiling to 99% - if (confidenceTotal >= 1.0) confidenceTotal = 0.99 - // have to do some weird conversion stuff to keep the decimals correct - confidenceTotal = Number.parseFloat(confidenceTotal.toFixed(3)) - //update to current matched authors before proceeding with next tests - let publicationAuthorMap = thisConf.getPublicationAuthorMap(publicationCsl) - const newTest = { - author: testAuthor, - confirmedAuthors: confirmedAuthors[personPublication['publication']['doi']], - confidenceItems: passedConfidenceTestsWithConf, - persons_publications_id: personPublication['id'], - doi: personPublication['publication']['doi'], - prevConf: personPublication['confidence'], - newConf: confidenceTotal - }; - if (confidenceTotal === personPublication['confidence']) { - passedTests.push(newTest) - } else if (confidenceTotal > personPublication['confidence']) { - warningTests.push(newTest) - } else { - failedTests.push(newTest) - } - }, {concurrency: 10}) - console.log(`Exiting loop 2 Test Author: ${testAuthor['names'][0]['lastName']}`) - }, { concurrency: 1}) + }) + // set ceiling to 99% + if (confidenceTotal >= 1.0) confidenceTotal = 0.99 + // have to do some weird conversion stuff to keep the decimals correct + confidenceTotal = Number.parseFloat(confidenceTotal.toFixed(3)) + //update to current matched authors before proceeding with next tests + let publicationAuthorMap = thisConf.getPublicationAuthorMap(publicationCsl) + const newTest = { + author: testAuthor, + confirmedAuthors: confirmedAuthors[personPublication['publication']['doi']], + confidenceItems: passedConfidenceTestsWithConf, + persons_publications_id: personPublication['id'], + doi: personPublication['publication']['doi'], + prevConf: personPublication['confidence'], + newConf: confidenceTotal + }; + if (confidenceTotal === personPublication['confidence']) { + passedTests.push(newTest) + } else if (confidenceTotal > personPublication['confidence']) { + warningTests.push(newTest) + } else { + failedTests.push(newTest) + } + }, {concurrency: 10}) + console.log(`Exiting loop 2 Test Author: ${testAuthor['names'][0]['lastName']}`) }, { concurrency: 1 }) console.log('Exited loop 1') diff --git a/ingest/updateConfidenceReviewStates.ts b/ingest/updateConfidenceReviewStates.ts index f570585f..e6e1dc17 100644 --- a/ingest/updateConfidenceReviewStates.ts +++ b/ingest/updateConfidenceReviewStates.ts @@ -82,26 +82,23 @@ async function main() { // testAuthors2.push(_.find(testAuthors, (testAuthor) => { return testAuthor['id']===78})) // testAuthors2.push(_.find(testAuthors, (testAuthor) => { return testAuthor['id']===48})) // testAuthors2.push(_.find(testAuthors, (testAuthor) => { return testAuthor['id']===61})) - testAuthors2.push(_.find(testAuthors, (testAuthor) => { return testAuthor['id']===197})) + testAuthors2.push(_.find(testAuthors, (testAuthor) => { return testAuthor['id']===120})) // console.log(`Test authors: ${JSON.stringify(testAuthors2, null, 2)}`) // get where last confidence test left off + // get all person publications without a confidence set + const lastConfidenceSet = await calculateConfidence.getLastPersonPubConfidenceSet() - let mostRecentPersonPubId = undefined - if (lastConfidenceSet) { - // mostRecentPersonPubId = 11145 - mostRecentPersonPubId = lastConfidenceSet.persons_publications_id - console.log(`Last Person Pub Confidence set is: ${mostRecentPersonPubId}`) - } else { - console.log(`Last Person Pub Confidence set is undefined`) - } + const overWriteExisting = false + + console.log(`Overwrite existing confidence sets: ${overWriteExisting}`) // const publicationYear = 2020 const publicationYear = undefined // break up authors into groups of 20 const testAuthorGroups = _.chunk(testAuthors, 10) await pMap (testAuthorGroups, async (authors, index) => { - const confidenceTests = await calculateConfidence.calculateConfidence (mostRecentPersonPubId, authors, (confirmedAuthorsByDoi || {}), publicationYear) + const confidenceTests = await calculateConfidence.calculateConfidence (authors, (confirmedAuthorsByDoi || {}), overWriteExisting, publicationYear) // next need to write checks found to DB and then calculate confidence accordingly let errorsInsert = [] From f54fd5d1f6b99b198c4f2c69d4dfff33916824c0 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 2 Aug 2021 23:56:12 -0400 Subject: [PATCH 26/43] fix counts off when new conf value goes under 0.5 --- .../readPersonsByInstitutionByYearAllCenters.js | 11 +++++++++++ ...adPersonsByInstitutionByYearByOrganization.js | 11 +++++++++++ client/src/pages/Index.vue | 16 +++++++++++----- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/client/src/gql/readPersonsByInstitutionByYearAllCenters.js b/client/src/gql/readPersonsByInstitutionByYearAllCenters.js index 56d1c11d..0306e091 100644 --- a/client/src/gql/readPersonsByInstitutionByYearAllCenters.js +++ b/client/src/gql/readPersonsByInstitutionByYearAllCenters.js @@ -78,6 +78,17 @@ export default function readPersonsByInstitutionByYearAllCenters (institutionNam value year } + confidencesets_persons_publications_aggregate(distinct_on: title, order_by: {title: asc, datetime: desc}, where: {year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}}}) { + nodes { + datetime + doi + id + publication_id + value + version + title + } + } reviews_persons_publications_aggregate(distinct_on: title, order_by: {title: asc, datetime: desc}) { aggregate { count(columns: title) diff --git a/client/src/gql/readPersonsByInstitutionByYearByOrganization.js b/client/src/gql/readPersonsByInstitutionByYearByOrganization.js index 001aded5..2ed1b324 100644 --- a/client/src/gql/readPersonsByInstitutionByYearByOrganization.js +++ b/client/src/gql/readPersonsByInstitutionByYearByOrganization.js @@ -64,6 +64,17 @@ export default function readPersonsByInstitutionByYearByOrganization (organizati title year } + confidencesets_persons_publications_aggregate(distinct_on: title, order_by: {title: asc, datetime: desc}, where: {year: {_gte: ${pubYearMin}, _lte: ${pubYearMax}}}) { + nodes { + datetime + doi + id + publication_id + value + version + title + } + } reviews_persons_publications( distinct_on: title, order_by: { diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 1b042ed8..6167c1e8 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -844,11 +844,17 @@ export default { } }, - getPersonPublicationCount (person) { + getPersonPublicationCount (person, minConfidence) { + let includeCount = 0 + _.each(person.confidencesets_persons_publications_aggregate.nodes, (node) => { + if (node.value >= 0.50) { + includeCount += 1 + } + }) if (this.selectedPersonTotal === 'All') { - return person.confidencesets_persons_publications.length + return includeCount } else { - return person.confidencesets_persons_publications.length - this.personReviewedPubCounts[person.id] + return includeCount - this.personReviewedPubCounts[person.id] } }, async loadPersonsWithFilter () { @@ -905,7 +911,7 @@ export default { // set the pub counts for person this.people = _.map(this.people, (person) => { - return _.set(person, 'person_publication_count', this.getPersonPublicationCount(person)) + return _.set(person, 'person_publication_count', this.getPersonPublicationCount(person, minConfidence)) }) // apply any sorting applied @@ -916,7 +922,7 @@ export default { // need to sort by total and then name, not guaranteed to be in order from what is returned from DB // first group items by count const peopleByCounts = await _.groupBy(this.people, (person) => { - return this.getPersonPublicationCount(person) + return person.person_publication_count }) console.log(`People by counts are: ${JSON.stringify(peopleByCounts, null, 2)}`) From a6c5ef1c10a6a25516400ae1403a3d70a3b33ad1 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 3 Aug 2021 09:01:20 -0400 Subject: [PATCH 27/43] force pending count to zero if ends up being neg # --- client/src/pages/Index.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 6167c1e8..bf2de8c4 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -854,7 +854,8 @@ export default { if (this.selectedPersonTotal === 'All') { return includeCount } else { - return includeCount - this.personReviewedPubCounts[person.id] + let pendingCount = includeCount - this.personReviewedPubCounts[person.id] + return (pendingCount >= 0 ? pendingCount : 0) } }, async loadPersonsWithFilter () { From 6caa734ec0d2d055e1bab78b31dc37f654a531fe Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 3 Aug 2021 09:02:36 -0400 Subject: [PATCH 28/43] fix spacing in index.vue --- client/src/pages/Index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index bf2de8c4..10b062c6 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -855,7 +855,7 @@ export default { return includeCount } else { let pendingCount = includeCount - this.personReviewedPubCounts[person.id] - return (pendingCount >= 0 ? pendingCount : 0) + return (pendingCount >= 0 ? pendingCount : 0) } }, async loadPersonsWithFilter () { From 194d81dd42dffdbe0fc4c0f1ad611f720fad74a6 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Wed, 4 Aug 2021 21:06:37 -0400 Subject: [PATCH 29/43] update synchronize reviews to use normalized title --- ingest/synchronizeReviewStates.ts | 38 ++++++++++++++++++------------- ingest/units/normalizer.ts | 2 +- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ingest/synchronizeReviewStates.ts b/ingest/synchronizeReviewStates.ts index ab6c8c93..dba90610 100644 --- a/ingest/synchronizeReviewStates.ts +++ b/ingest/synchronizeReviewStates.ts @@ -7,7 +7,7 @@ import fetch from 'node-fetch' import dotenv from 'dotenv' import pMap from 'p-map' import { CalculateConfidence } from './modules/calculateConfidence' - +import { normalizeString } from './units/normalizer' import readReviewTypes from './gql/readReviewTypes' import insertReview from '../client/src/gql/insertReview' import readPersonPublicationsReviews from './gql/readPersonPublicationsReviews' @@ -50,6 +50,11 @@ async function loadOrganizations () { return reviewOrgsResult.data.review_organization } +function getPublicationTitleKey (title: string) { + // normalize the string and remove characters like dashes as well + return normalizeString(title) +} + async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { const reviewStates = await loadReviewStates() @@ -86,14 +91,15 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { _.each(reviewStates, (reviewType) => { const publications = publicationsGroupedByReview[reviewType] _.each(publications, (personPub) => { - if (personPub.doi !== null){ - if (!publicationTitlesByReviewType[personPub['publication'].title]) { - publicationTitlesByReviewType[personPub['publication'].title] = {} + if (personPub['publication'].title !== null){ + const titleKey = getPublicationTitleKey(personPub['publication'].title) + if (!publicationTitlesByReviewType[titleKey]) { + publicationTitlesByReviewType[titleKey] = {} } - if (!publicationTitlesByReviewType[personPub['publication'].title][reviewType]) { - publicationTitlesByReviewType[personPub['publication'].title][reviewType] = [] + if (!publicationTitlesByReviewType[titleKey][reviewType]) { + publicationTitlesByReviewType[titleKey][reviewType] = [] } - publicationTitlesByReviewType[personPub['publication'].title][reviewType].push(personPub) + publicationTitlesByReviewType[titleKey][reviewType].push(personPub) } }) }) @@ -101,17 +107,17 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { // check for any title's with reviews out of sync const publicationTitlesOutOfSync = [] - _.each(_.keys(publicationTitlesByReviewType), (title) => { - if (_.keys(publicationTitlesByReviewType[title]).length > 1) { + _.each(_.keys(publicationTitlesByReviewType), (titleKey) => { + if (_.keys(publicationTitlesByReviewType[titleKey]).length > 1) { // console.log(`Warning: Doi out of sync found: ${doi} for person id: ${this.person.id} doi record: ${JSON.stringify(publicationDoisByReviewType[doi], null, 2)}`) - publicationTitlesOutOfSync.push(title) + publicationTitlesOutOfSync.push(titleKey) } }) if (publicationTitlesOutOfSync.length > 0) { console.log(`Person id '${person['id']}' Dois found with reviews out of sync: ${JSON.stringify(publicationTitlesOutOfSync, null, 2)}`) console.log(`Synchronizing Reviews for these personPubs out of sync for person id '${person['id']}'...`) - await pMap(publicationTitlesOutOfSync, async (title) => { + await pMap(publicationTitlesOutOfSync, async (titleKey) => { // console.log(`Publication title '${title}' reviews by review type: ${JSON.stringify(publicationTitlesByReviewType[title], null, 2)}`) // get most recent review let mostRecentUpdateTime = undefined @@ -119,9 +125,9 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { let mostRecentReview = undefined let newReviewStatus = undefined let newReviewOrgValue = undefined - _.each(_.keys(publicationTitlesByReviewType[title]), (reviewType) => { + _.each(_.keys(publicationTitlesByReviewType[titleKey]), (reviewType) => { // just get first one - const personPub = publicationTitlesByReviewType[title][reviewType][0] + const personPub = publicationTitlesByReviewType[titleKey][reviewType][0] if (personPub['reviews_aggregate'].nodes.length > 0) { // console.log(`Looking at reviews for doi: ${doi}, reviewType: ${reviewType}, nodes: ${JSON.stringify(personPub['reviews_aggregate'].nodes, null, 2)}`) const currentDateTime = new Date(personPub['reviews_aggregate'].nodes[0].datetime) @@ -137,9 +143,9 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { }) if (mostRecentReview) { await pMap (reviewStates, async (reviewType) => { - if (reviewType !== mostRecentReview.reviewType && publicationTitlesByReviewType[title][reviewType]) { - await pMap (publicationTitlesByReviewType[title][reviewType], async (personPub) => { - console.log(`Inserting Review for title: ${title}, personpub: '${personPub['id']}', most recent review ${JSON.stringify(mostRecentReview, null, 2)}`) + if (reviewType !== mostRecentReview.reviewType && publicationTitlesByReviewType[titleKey][reviewType]) { + await pMap (publicationTitlesByReviewType[titleKey][reviewType], async (personPub) => { + console.log(`Inserting Review for title key: ${titleKey}, personpub: '${personPub['id']}', most recent review ${JSON.stringify(mostRecentReview, null, 2)}`) const mutateResult = await client.mutate( insertReview(mostRecentReview.userId, personPub['id'], mostRecentReview.reviewType, mostRecentReview.reviewOrgValue) ) diff --git a/ingest/units/normalizer.ts b/ingest/units/normalizer.ts index a323665e..b221cd73 100644 --- a/ingest/units/normalizer.ts +++ b/ingest/units/normalizer.ts @@ -46,7 +46,7 @@ export function normalizeString(value, options = {}) { .normalize('NFD') .replace(/[\u0300-\u036f]/g, '') // Remove diacritics .replace(/[\u2019]/g, '\u0027') // the u0027 also normalizes the curly apostrophe to the straight one - .replace(/[&\/\\#,+()$~%.'":*?<>{}!]/g,'') // remove periods and other remaining special characters + .replace(/[&\/\\#,+()$~%.'":*?<>{}!-]/g,'') // remove periods and other remaining special characters if (!skipLower) { newValue = _.lowerCase(newValue) From e29cc7801ce9f7f7076cb585eb134857b9be852a Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 9 Aug 2021 11:57:56 -0400 Subject: [PATCH 30/43] group publications into sets by doi and title --- client/src/pages/Index.vue | 438 ++++++++++++++++++++++++++++++------- 1 file changed, 355 insertions(+), 83 deletions(-) diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 10b062c6..e96c2480 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -140,49 +140,50 @@ - + Move To: @@ -210,7 +211,7 @@ dense no-wrap size="md" - v-for="(personPub, index) in getSortedPersonPublicationsBySourceName(publicationsGroupedByTitleByReview[reviewTypeFilter][personPublication.publication.title])" + v-for="(personPub, index) in getSortedPersonPublicationsBySourceName(getPersonPubSet(getPersonPubSetId(personPublication.id)).personPublications)" :key="index" :color="getSourceNameChipColor(personPub.publication.source_name)" text-color="white" @@ -447,9 +448,20 @@ export default { publicationsGroupedByReview: {}, personPublicationsCombinedMatches: [], personReviewedPubCounts: {}, + // these are helper objects to connect personPubSets together + // PersonPubId -> Person Pub Set ID Pointers + personPubSetPointer: {}, + // PersonPubSet Id -> Person Pub Id list (i.e., the set itself) + personPubSetsById: {}, + // the current index for personPubSets, will increment whenever adding a new one + personPubSetIdIndex: 0, + personPublicationsById: {}, + personPubSetsByReviewType: {}, personPublicationsCombinedMatchesByReview: {}, - filteredPersonPublicationsCombinedMatchesByReview: {}, publicationsGroupedByTitleByReview: {}, + publicationsGroupedByDoiByReview: {}, + // will be a personsPublication id mapped to an object of {titleKey. doiKey} + personPublicationsKeys: {}, publicationJournalClassifications: [], institutions: [], institutionGroup: [], @@ -559,6 +571,181 @@ export default { } }, methods: { + getPersonPublicationsInSet (pubSet) { + return _.map(pubSet.personPublicationIds, (pubId) => { + return this.getPersonPublicationById(pubId) + }) + }, + getMainPersonPubFromSet (pubSet) { + return pubSet.mainPersonPub + }, + // will link all personPubs in this list together + linkPersonPubs (personPubList, reviewType) { + // link all person pubs together in this list + const totalPubs = personPubList.length + _.each(personPubList, (personPub, index) => { + // at last item do nothing + try { + if (index === 0 && totalPubs === 1) { + console.log(`Linking index: ${index}, only one pub in set, personpub: ${JSON.stringify(personPub['id'], null, 2)}`) + // will start a new personpub set list if not already in one + console.log(`Passing person pub id to start a new set: ${personPub['id']}`) + this.startPersonPubSet(personPub['id'], reviewType) + } else if (index !== (totalPubs - 1)) { + console.log(`Linking index: ${index} to ${index + 1} of ${totalPubs}), personpub: ${JSON.stringify(personPub['id'], null, 2)}`) + const nextPersonPub = _.nth(personPubList, (index + 1)) + this.linkPersonPubPair(personPub.id, nextPersonPub.id, reviewType) + } + } catch (error) { + console.log(`Warning, error on linking publications: ${error}`) + throw error + } + }) + }, + // this method will link person pubs by putting them in a person pub set + // together. If neither are already in a person pub set, they will be grouped together in a new set + // If only one has a set, the other will be added to that set + // If both are currently within a set, it will merge the two sets together + linkPersonPubPair (personPub1Id, personPub2Id, reviewType) { + const notInPersonPub1SetId = this.notInPersonPubSet(personPub1Id) + const notInPersonPub2SetId = this.notInPersonPubSet(personPub2Id) + const personPubSet1Id = this.getPersonPubSetId(personPub1Id) + const personPubSet2Id = this.getPersonPubSetId(personPub2Id) + console.log(`Linking person pub id 1: ${personPub1Id} to person pub id 2: ${personPub2Id}`) + if (notInPersonPub1SetId && notInPersonPub2SetId) { + // neither one is in a set yet and just add to set list + console.log(`Starting new set for personPubId1: ${personPub1Id}`) + const newSetId = this.startPersonPubSet(personPub1Id, reviewType) + console.log(`Created personPubset1: ${newSetId} and added person pub1: ${personPub1Id}`) + console.log(`Adding personPubId2: ${personPub2Id} to set1id: ${newSetId}`) + this.addPersonPubToSet(newSetId, personPub2Id, reviewType) + } else if (notInPersonPub2SetId) { + console.log(`Adding personPubId2: ${personPub2Id} to set1id: ${personPubSet1Id}`) + this.addPersonPubToSet(personPubSet1Id, personPub2Id, reviewType) + } else if (notInPersonPub1SetId) { + console.log(`Adding personPubId1: ${personPub1Id} to set2id: ${personPubSet2Id}`) + this.addPersonPubToSet(personPubSet2Id, personPub1Id, reviewType) + } else { + // they are both in existing sets and need to merge them + // do nothing if they have the same pubsetid as they are already + // in the same set + console.log(`Merging sets for personPubId1: ${personPub1Id} and personPubId2: ${personPub2Id}, set2Id->set1Id: (${personPubSet2Id}->${personPubSet1Id})`) + this.mergePersonPubSets(personPubSet1Id, personPubSet2Id, reviewType) + } + }, + mergePersonPubSets (set1Id, set2Id, reviewType) { + // do nothing if they are the same set id + console.log(`Merging person pub sets set1: ${set1Id} set2: ${set2Id}`) + if (set1Id !== set2Id) { + // add items from set2 into set1 if not already there, assumes everything is up to date with pointers + const set1 = this.getPersonPubSet(set1Id) + const set2 = this.getPersonPubSet(set2Id) + if (set1.reviewType !== reviewType || set2.reviewType !== reviewType) { + const error = `Error: Mismatch in reviewType for sets to be merged. Expected: ${reviewType}, found set 1: ${set1.reviewType} set 2: ${set2.reviewType}` + console.log(error) + throw error + } + const set2List = set2.personPublicationIds + _.each(set2List, (personPubId) => { + this.addPersonPubToSet(set1Id, personPubId, reviewType) + }) + // destroy the set2List + this.removePersonPubSet(set2Id) + } + }, + removePersonPubSet (setId) { + // only works if personPubs in this set already pointing to another one, else throw error + // do nothing if set already gone + const set = this.getPersonPubSet(setId) + if (set) { + _.each(set, (personPubId) => { + if (setId && this.getPersonPubSetId(personPubId) === setId) { + const error = `Cannot remove person Pub Set (on merge), personPubId: ${personPubId} not in any other set` + console.log(error) + throw error + } + }) + // if we get this far no errors encountered, and all person pubs are now in another set + // go ahead and delete it + _.unset(this.personPubSetsById, setId) + } + }, + addPersonPubToSet (setId, personPubId, reviewType) { + // proceed if set exists + const set = this.getPersonPubSet(setId) + if (set) { + // do nothing if already in the set + if (this.getPersonPubSetId(personPubId) !== setId) { + if (set.reviewType !== reviewType) { + const error = `Failed to add person pub to set with mismatched review types. Expected ${reviewType}, found: ${set.reviewType}` + console.log(error) + throw error + } + const addPub = this.getPersonPublicationById(personPubId) + this.personPubSetsById[setId].personPublicationIds = _.concat(this.personPubSetsById[setId].personPublicationIds, personPubId) + this.personPubSetsById[setId].personPublications = _.concat(this.personPubSetsById[setId].personPublications, addPub) + this.personPubSetPointer[personPubId] = setId + // console.log(`After add personpub: ${personPubId} to setid: ${setId} pubset pointers are: ${JSON.stringify(this.personPubSetPointer, null, 2)}`) + const mainPersonPub = this.getPersonPublicationById(set.mainPersonPubId) + if (!set.mainPersonPubId || this.getPublicationConfidence(mainPersonPub) < this.getPublicationConfidence(addPub)) { + _.set(set, 'mainPersonPub', addPub) + _.set(set, 'mainPersonPubId', addPub.id) + } + } + } else { + const error = `Failed to add personPub with id: ${personPubId} to set id: ${setId}, personPubSet does not exist` + console.log(error) + throw error + } + }, + notInPersonPubSet (personPubId) { + // true if already in a set + return !this.personPubSetPointer[personPubId] + }, + getPersonPubSet (setId) { + return this.personPubSetsById[setId] + }, + // this method is not currently thread-safe + // creates a new person pub set if one does not already exist for the given + // person Pub Id + startPersonPubSet (personPubId, reviewType) { + if (this.notInPersonPubSet(personPubId)) { + console.log(`Creating person pub set for pub id: ${personPubId}`) + const personPubSetId = this.getNextPersonPubSetId() + this.personPubSetPointer[personPubId] = personPubSetId + const personPub = this.getPersonPublicationById(personPubId) + this.personPubSetsById[personPubSetId] = { + personPublicationIds: [personPubId], + personPublications: [personPub], + mainPersonPubId: personPubId, + mainPersonPub: personPub, + reviewType: reviewType + } + return personPubSetId + } else { + const currentSetId = this.getPersonPubSetId(personPubId) + const currentSet = this.getPersonPubSet(currentSetId) + if (currentSet.reviewType !== reviewType) { + const error = `Error: Mismatch on review type for person Pub set for personPub id: ${personPubId}, expected review type: ${reviewType} and found review type: ${currentSet.reviewType}` + console.log(error) + throw error + } else { + return this.getPersonPubSetId(personPubId) + } + } + }, + getPersonPublicationById (personPubId) { + return this.personPublicationsById[personPubId] + }, + // returns a person Pub set if it exists for that personPub, else returns undefined + getPersonPubSetId (personPubId) { + return this.personPubSetPointer[personPubId] + }, + // this method is not currently thread-safe + getNextPersonPubSetId () { + this.personPubSetIdIndex += 1 + return this.personPubSetIdIndex + }, getPublicationDoiKey (publication) { if (!publication.doi) { return `${publication.source_name}_${publication.source_id}` @@ -669,8 +856,8 @@ export default { }, // depending on the source return source uri getSourceUri (personPublication) { - console.log(`Getting source uri for personPublication pub`) - console.log(`Process env wos url: ${process.env.WOS_PUBLICATION_URL}`) + // console.log(`Getting source uri for personPublication pub`) + // console.log(`Process env wos url: ${process.env.WOS_PUBLICATION_URL}`) const sourceId = this.getPublicationSourceId(personPublication) if (sourceId) { if (personPublication.publication.source_name.toLowerCase() === 'scopus') { @@ -682,7 +869,7 @@ export default { } else if (personPublication.publication.source_name.toLowerCase() === 'webofscience') { return this.getWebOfScienceUri(sourceId) } else if (personPublication.publication.source_name.toLowerCase() === 'semanticscholar') { - console.log(`got semantic scholar paper uri ${this.getSemanticScholarUri(sourceId)}`) + // console.log(`got semantic scholar paper uri ${this.getSemanticScholarUri(sourceId)}`) return this.getSemanticScholarUri(sourceId) } } else { @@ -702,6 +889,9 @@ export default { getWebOfScienceUri (wosId) { return `${process.env.WOS_PUBLICATION_URL}${wosId}` }, + getPublicationsGroupedByReviewCount (reviewType) { + return this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] ? this.filteredPersonPublicationsCombinedMatchesByReview[reviewType].length : 0 + }, getPublicationsGroupedByTitleByReviewCount (reviewType) { return this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] ? this.filteredPersonPublicationsCombinedMatchesByReview[reviewType].length : 0 }, @@ -792,7 +982,7 @@ export default { this.selectedReviewState = reviewState }, async scrollToPublication (index) { - console.log(`updating scroll ${index} for ${this.selectedReviewState} ${this.$refs['pubScroll'].toString}`) + // console.log(`updating scroll ${index} for ${this.selectedReviewState} ${this.$refs['pubScroll'].toString}`) this.$refs['pubScroll'].scrollTo(index + 1) }, async showCurrentSelectedPublication () { @@ -846,9 +1036,18 @@ export default { getPersonPublicationCount (person, minConfidence) { let includeCount = 0 + let titles = {} + let dois = {} _.each(person.confidencesets_persons_publications_aggregate.nodes, (node) => { - if (node.value >= 0.50) { - includeCount += 1 + const title = node.title + const titleKey = this.getPublicationTitleKey(title) + const doi = _.toLower(node.doi) + if (node.value >= minConfidence) { + if (!titles[titleKey] && (doi === null || !dois[doi])) { + titles[titleKey] = 1 + if (doi != null) dois[doi] = doi + includeCount += 1 + } } }) if (this.selectedPersonTotal === 'All') { @@ -856,6 +1055,7 @@ export default { } else { let pendingCount = includeCount - this.personReviewedPubCounts[person.id] return (pendingCount >= 0 ? pendingCount : 0) + // return pendingCount } }, async loadPersonsWithFilter () { @@ -890,9 +1090,11 @@ export default { console.log('Checking for reviewed pub counts...') _.each(this.people, (person) => { const reviewedTitles = {} + const reviewedDOIs = {} _.each(person.reviews_persons_publications, (review) => { if (review.review_type && review.review_type !== 'pending') { - reviewedTitles[review.title] = review + reviewedTitles[this.getPublicationTitleKey(review.title)] = review + if (review.doi) reviewedDOIs[_.toLower(review.doi)] = review.doi } }) @@ -900,7 +1102,7 @@ export default { let filteredReviewedTitlesCount = 0 _.each(person.confidencesets_persons_publications, (confidenceSet) => { const title = confidenceSet.title - if (reviewedTitles[title]) { + if (reviewedTitles[this.getPublicationTitleKey(title)]) { // } || reviewedDOIs[_.toLower(confidenceSet.doi)]) { filteredReviewedTitlesCount += 1 } }) @@ -926,7 +1128,7 @@ export default { return person.person_publication_count }) - console.log(`People by counts are: ${JSON.stringify(peopleByCounts, null, 2)}`) + // console.log(`People by counts are: ${JSON.stringify(peopleByCounts, null, 2)}`) // sort each person array by name for each count const peopleByCountsByName = await _.mapValues(peopleByCounts, (persons) => { @@ -975,17 +1177,17 @@ export default { const publicationId = personPublication.publication.id const result = await this.$apollo.query(readAuthorsByPublication(publicationId)) this.publicationAuthors = result.data.publications_authors - console.log(`Loaded Publication Authors: ${JSON.stringify(this.publicationAuthors)}`) + // console.log(`Loaded Publication Authors: ${JSON.stringify(this.publicationAuthors)}`) // load up author positions of possible matches this.matchedPublicationAuthors = _.filter(this.publicationAuthors, function (author) { return author.family_name.toLowerCase() === personPublication.person.family_name.toLowerCase() }) - console.log(`Matched authors are: ${JSON.stringify(this.matchedPublicationAuthors, null, 2)}`) + // console.log(`Matched authors are: ${JSON.stringify(this.matchedPublicationAuthors, null, 2)}`) }, async loadConfidenceSet (personPublication) { this.confidenceSetItems = [] this.confidenceSet = undefined - console.log(`Trying to load confidence sets for pub: ${JSON.stringify(personPublication, null, 2)}`) + // console.log(`Trying to load confidence sets for pub: ${JSON.stringify(personPublication, null, 2)}`) if (personPublication.confidencesets_aggregate && personPublication.confidencesets_aggregate.nodes.length > 0) { this.confidenceSet = personPublication.confidencesets_aggregate.nodes[0] @@ -993,7 +1195,7 @@ export default { const result = await this.$apollo.query(readConfidenceSetItems(this.confidenceSet.id)) this.confidenceSetItems = result.data.confidencesets_items this.confidenceSetItems = _.transform(this.confidenceSetItems, (result, setItem) => { - console.log(`Trying to set properties for confidence set item: ${JSON.stringify(setItem, null, 2)}`) + // console.log(`Trying to set properties for confidence set item: ${JSON.stringify(setItem, null, 2)}`) _.set(setItem, 'confidence_type_name', setItem.confidence_type.name) _.set(setItem, 'confidence_type_rank', setItem.confidence_type.rank) _.set(setItem, 'confidence_type_desc', setItem.confidence_type.description) @@ -1025,7 +1227,16 @@ export default { this.personPublicationsCombinedMatchesByReview = {} this.filteredPersonPublicationsCombinedMatchesByReview = {} this.publicationsGroupedByTitleByReview = {} - this.publicationsGroupedByTitle = {} + this.publicationsGroupedByDoiByReview = {} + this.personPubSetPointer = {} + this.personPubSetsById = {} + this.personPubSetIdIndex = 0 + this.personPublicationsById = {} + this.personPubSetsByReviewType = {} + this.personPublicationsCombinedMatchesByReview = {} + this.publicationsGroupedByTitleByReview = {} + this.publicationsGroupedByDoiByReview = {} + this.personPublicationsKeys = {} this.confidenceSetItems = [] this.confidenceSet = undefined }, @@ -1045,7 +1256,10 @@ export default { async loadPersonPublicationsCombinedMatches () { // this.fundersByDoi = {} console.log(`Start group by publications for person id: ${this.person.id} ${moment().format('HH:mm:ss:SSS')}`) + const indexThis = this this.publicationsGroupedByReview = _.groupBy(this.publications, function (pub) { + if (!indexThis.personPublicationsById) indexThis.personPublicationsById = {} + indexThis.personPublicationsById[pub.id] = pub if (pub.reviews_aggregate.nodes && pub.reviews_aggregate.nodes.length > 0) { return pub.reviews_aggregate.nodes[0].review_type } else { @@ -1056,46 +1270,91 @@ export default { // check for any doi's with reviews out of sync, // if more than one review type found add doi mapped to array of reviewtype to array pub list + // map both by shared title and by shared doi and merged lists together later let publicationTitlesByReviewType = {} + let publicationDoisByReviewType = {} // put in pubs grouped by doi for each review status _.each(this.reviewStates, (reviewType) => { const publications = this.publicationsGroupedByReview[reviewType] + + // seed the personPublication keys + _.each(publications, (personPub) => { + this.personPublicationsKeys[personPub.id] = { + titleKey: this.getPublicationTitleKey(personPub.publication.title), + doiKey: this.getPublicationTitleKey(personPub.publication.doi) + } + }) this.publicationsGroupedByTitleByReview[reviewType] = _.groupBy(publications, (personPub) => { - let title = personPub.publication.title - if (!publicationTitlesByReviewType[title]) { - publicationTitlesByReviewType[title] = {} + // let title = personPub.publication.title + const titleKey = this.personPublicationsKeys[personPub.id].titleKey + // console.log(`Title: '${title} title key: ${titleKey}'`) + if (!publicationTitlesByReviewType[titleKey]) { + publicationTitlesByReviewType[titleKey] = {} + } + if (!publicationTitlesByReviewType[titleKey][reviewType]) { + publicationTitlesByReviewType[titleKey][reviewType] = reviewType + } + // publicationTitlesByReviewType[titleKey][reviewType].push(personPub) + return `${titleKey}` + }) + + this.publicationsGroupedByDoiByReview[reviewType] = _.groupBy(publications, (personPub) => { + // let doi = personPub.publication.doi + const doiKey = this.personPublicationsKeys[personPub.id].doiKey + // console.log(`Doi: '${doi} doi key: ${doiKey}'`) + if (!publicationDoisByReviewType[doiKey]) { + publicationDoisByReviewType[doiKey] = {} } - if (!publicationTitlesByReviewType[title][reviewType]) { - publicationTitlesByReviewType[title][reviewType] = [] + if (!publicationDoisByReviewType[doiKey][reviewType]) { + publicationDoisByReviewType[doiKey][reviewType] = reviewType } - publicationTitlesByReviewType[title][reviewType].push(personPub) - return `${title}` + // publicationDoisByReviewType[doiKey][reviewType].push(personPub) + return `${doiKey}` }) // console.log(`Person pubs grouped by Title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) // grab one with highest confidence to display and grab others via title later when changing status - this.personPublicationsCombinedMatchesByReview[reviewType] = _.map(_.keys(this.publicationsGroupedByTitleByReview[reviewType]), (title) => { - // get match with highest confidence level and use that one - const personPubs = this.publicationsGroupedByTitleByReview[reviewType][title] - let currentPersonPub - _.each(personPubs, (personPub, index) => { - if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { - currentPersonPub = personPub - } - }) - return currentPersonPub + + // this.personPublicationsCombinedMatchesByReview[reviewType] = {} + + // keep a map of personPubId to set id in order to find the set that something should be added to if found as same pub + // merge personPubs together by title and then doi + _.each(_.keys(this.publicationsGroupedByTitleByReview[reviewType]), (titleKey) => { + this.linkPersonPubs(this.publicationsGroupedByTitleByReview[reviewType][titleKey], reviewType) + }) + + // now link together if same doi (if already found above will add to existing set) + _.each(_.keys(this.publicationsGroupedByDoiByReview[reviewType]), (doiKey) => { + this.linkPersonPubs(this.publicationsGroupedByDoiByReview[reviewType][doiKey], reviewType) }) + + // get match with highest confidence level and use that one + // let currentPersonPub + // _.each(personPubs, (personPub, index) => { + // if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { + // currentPersonPub = personPub + // } + // }) + // return currentPersonPub + // }) + }) + + // group pub sets by review type + this.personPubSetsByReviewType = _.groupBy(_.values(this.personPubSetsById), (pubSet) => { + return pubSet.reviewType }) + // console.log(`PersonPublication Sets by Review Type are: ${JSON.stringify(this.personPubSetsByReviewType, null, 2)}`) + // console.log(`Publications grouped by title by review: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) // check for any doi's with reviews out of sync const publicationTitlesOutOfSync = [] - _.each(_.keys(publicationTitlesByReviewType), (title) => { - if (_.keys(publicationTitlesByReviewType[title]).length > 1) { - console.log(`Warning: Title out of sync found: ${title} for person id: ${this.person.id} title record: ${JSON.stringify(publicationTitlesByReviewType[title], null, 2)}`) - publicationTitlesOutOfSync.push(title) + _.each(_.keys(publicationTitlesByReviewType), (titleKey) => { + if (_.keys(publicationTitlesByReviewType[titleKey]).length > 1) { + console.log(`Warning: Title out of sync found: ${titleKey} for person id: ${this.person.id} title record: ${JSON.stringify(publicationTitlesByReviewType[titleKey], null, 2)}`) + publicationTitlesOutOfSync.push(titleKey) } }) @@ -1107,23 +1366,51 @@ export default { this.setCurrentPersonPublicationsCombinedMatches() // console.log(`Funders by Doi ${JSON.stringify(_.keys(this.fundersByDoi).length, null, 2)}`) }, + removeSpaces (value) { + if (_.isString(value)) { + return _.clone(value).replace(/\s/g, '') + } else { + return value + } + }, + normalizeString (value, options = {}) { + if (_.isString(value)) { + let newValue = _.clone(value) + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') // Remove diacritics + .replace(/[\u2019]/g, '\u0027') // the u0027 also normalizes the curly apostrophe to the straight one + .replace(/[&\\#,+()$~%.'":*?<>{}!-]/g, '') // remove periods and other remaining special characters + + newValue = _.lowerCase(newValue) + const returningValue = newValue + return this.removeSpaces(returningValue) + } else { + return value + } + }, + getPublicationTitleKey (title) { + // normalize the string and remove characters like dashes as well + return this.normalizeString(title) + }, async filterPublications () { let filterOutCurrentPublication = false this.filteredPersonPublicationsCombinedMatchesByReview = _.mapValues( - this.personPublicationsCombinedMatchesByReview, - (personPublications) => { - return _.filter(personPublications, (item) => { - let includePublication = item.publication.title.toLowerCase().includes(this.pubSearch.toLowerCase().trim()) + this.personPubSetsByReviewType, + (personPubSets) => { + return _.filter(personPubSets, (pubSet) => { + const personPub = this.getMainPersonPubFromSet(pubSet) + // console.log(`Person pub on filter is: ${JSON.stringify(personPub, null, 2)}, person pub sets are: ${JSON.stringify(personPubSets, null, 2)}`) + let includePublication = personPub.publication.title.toLowerCase().includes(this.pubSearch.toLowerCase().trim()) if (includePublication) { // also check if confidence is to be filtered out // console.log('checking if we should include publication') // console.log(`confidence val is: ${this.getPublicationConfidence(item)}`) - if (this.selectedPersonConfidence === '50%' && this.getPublicationConfidence(item) < 0.50) { + if (this.selectedPersonConfidence === '50%' && this.getPublicationConfidence(personPub) < 0.50) { // console.log('trying to filter out publication') includePublication = false } } - if (!includePublication && this.personPublication && item.id === this.personPublication.id) { + if (!includePublication && this.personPublication && personPub.id === this.personPublication.id) { // clear out the publication from view if it is filtered out of the results filterOutCurrentPublication = true } @@ -1173,41 +1460,25 @@ export default { // apply any sorting applied console.log('sorting', this.selectedPersonPubSort) if (this.selectedPersonPubSort === 'Title') { - this.personPublicationsCombinedMatches = _.sortBy(this.personPublicationsCombinedMatches, (personPub) => { + this.personPublicationsCombinedMatches = _.sortBy(this.personPublicationsCombinedMatches, (personPubSet) => { + // console.log(`Placing in sort of personPubSet: ${JSON.stringify(personPubSet, null, 2)}`) + const personPub = this.getMainPersonPubFromSet(personPubSet) return this.trimFirstArticles(personPub.publication.title) }) - } else if (this.selectedPersonPubSort === 'Source') { - // need to sort by confidence and then name, not guaranteed to be in order from what is returned from DB - // first group items by count - const groupedPubs = _.groupBy(this.personPublicationsCombinedMatches, (pub) => { - return pub.publication.source_name - }) - - // sort each person array by title for each conf - const groupedPubsByTitle = _.mapValues(groupedPubs, (pubs) => { - return _.sortBy(pubs, ['title']) - }) - - // get array of pub values (i.e., keys) sorted in reverse - const sortedKeys = _.sortBy(_.keys(groupedPubsByTitle), (key) => { return key }) - - // now push values into array in desc order of count and flatten - let sortedPubs = [] - _.each(sortedKeys, (key) => { - sortedPubs.push(groupedPubsByTitle[key]) - }) - - this.personPublicationsCombinedMatches = _.flatten(sortedPubs) } else { // need to sort by confidence and then name, not guaranteed to be in order from what is returned from DB // first group items by count - const pubsByConf = _.groupBy(this.personPublicationsCombinedMatches, (pub) => { + const pubsByConf = _.groupBy(this.personPublicationsCombinedMatches, (pubSet) => { + const pub = this.getMainPersonPubFromSet(pubSet) return this.getPublicationConfidence(pub) }) // sort each person array by title for each conf - const pubsByConfByName = _.mapValues(pubsByConf, (pubs) => { - return _.sortBy(pubs, ['title']) + const pubsByConfByName = _.mapValues(pubsByConf, (pubSets) => { + return _.sortBy(pubSets, (pubSet) => { + const mainPersonPub = this.getMainPersonPubFromSet(pubSet) + return mainPersonPub.publication.title + }) }) // get array of confidence values (i.e., keys) sorted in reverse @@ -1280,12 +1551,12 @@ export default { this.publication = result.data.publications[0] _.set(this.publication, 'doi', _.toLower(this.publication.doi)) // console.log(`Loaded Publication: ${JSON.stringify(this.publication)}`) - console.log(`Publication journal is: ${JSON.stringify(this.publication.journal_title, null, 2)}`) + // console.log(`Publication journal is: ${JSON.stringify(this.publication.journal_title, null, 2)}`) this.publicationCitation = this.getCitationApa(this.publication.csl_string) this.publicationJournalClassifications = _.map(this.publication.journal.journals_classifications_aggregate.nodes, (node) => { return node.classification }) - console.log(`Found Journal Classifications: ${JSON.stringify(this.publicationJournalClassifications, null, 2)}`) + // console.log(`Found Journal Classifications: ${JSON.stringify(this.publicationJournalClassifications, null, 2)}`) try { const sanitizedDoi = sanitize(this.publication.doi, { replacement: '_' }) const imageHostBase = process.env.IMAGE_HOST_URL @@ -1319,7 +1590,8 @@ export default { this.person = person // add the review for personPublications with the same title in the list let title = personPublication.publication.title - const personPubs = this.publicationsGroupedByTitleByReview[this.reviewTypeFilter][title] + const titleKey = this.getPublicationTitleKey(title) + const personPubs = this.publicationsGroupedByTitleByReview[this.reviewTypeFilter][titleKey] try { let mutateResults = [] @@ -1335,7 +1607,7 @@ export default { Vue.delete(this.personPublicationsCombinedMatches, index) // transfer from one review queue to the next primarily for counts, other sorting will shake out on reload when clicking the tab // remove from current lists - _.unset(this.publicationsGroupedByTitleByReview[this.reviewTypeFilter], title) + _.unset(this.publicationsGroupedByTitleByReview[this.reviewTypeFilter], titleKey) _.remove(this.personPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (pub) => { return pub.id === personPub.id }) @@ -1343,7 +1615,7 @@ export default { return pub.id === personPub.id }) // add to new lists - this.publicationsGroupedByTitleByReview[reviewType][title] = personPubs + this.publicationsGroupedByTitleByReview[reviewType][titleKey] = personPubs this.personPublicationsCombinedMatchesByReview[reviewType].push(personPub) this.filteredPersonPublicationsCombinedMatchesByReview[reviewType].push(personPub) if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { From 951368dcb213dca936a245f55cf354a4ebd81512 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Mon, 9 Aug 2021 15:59:30 -0400 Subject: [PATCH 31/43] updated author review to work for pub sets --- client/src/pages/Index.vue | 89 +++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index e96c2480..60345a6b 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -114,10 +114,10 @@ v-model="reviewTypeFilter" dense > - - - - + + + + { // const personPub = personPubs[0] @@ -1606,41 +1603,45 @@ export default { this.$refs[`personPub${index}`].hide() Vue.delete(this.personPublicationsCombinedMatches, index) // transfer from one review queue to the next primarily for counts, other sorting will shake out on reload when clicking the tab - // remove from current lists - _.unset(this.publicationsGroupedByTitleByReview[this.reviewTypeFilter], titleKey) - _.remove(this.personPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (pub) => { - return pub.id === personPub.id - }) - _.remove(this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (pub) => { - return pub.id === personPub.id - }) - // add to new lists - this.publicationsGroupedByTitleByReview[reviewType][titleKey] = personPubs - this.personPublicationsCombinedMatchesByReview[reviewType].push(personPub) - this.filteredPersonPublicationsCombinedMatchesByReview[reviewType].push(personPub) - if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { - const currentPersonIndex = _.findIndex(this.people, (person) => { - return person.id === this.person.id - }) - this.personReviewedPubCounts[this.person.id] += 1 - this.people[currentPersonIndex].person_publication_count -= 1 - await this.changedPendingCounts(currentPersonIndex) - // this.people[currentPersonIndex].reviews_persons_publications_aggregate.aggregate.count = 1 - this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count -= 1 - } else if (this.selectedPersonTotal === 'Pending' && reviewType === 'pending') { - const currentPersonIndex = _.findIndex(this.people, (person) => { - return person.id === this.person.id - }) - this.personReviewedPubCounts[this.person.id] -= 1 - this.people[currentPersonIndex].person_publication_count += 1 - await this.changedPendingCounts(currentPersonIndex) - // this.people[currentPersonIndex].reviews_persons_publications_aggregate.aggregate.count += 1 - this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count += 1 - } } mutateResults.push(mutateResult) this.publicationsReloadPending = true }) + // remove set from related lists + this.personPubSetsByReviewType[this.reviewTypeFilter] = _.filter(this.personPubSetsByReviewType[this.reviewTypeFilter], (curPubSet) => { + return pubSet.mainPersonPub.id !== curPubSet.mainPersonPub.id + }) + this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter] = _.filter(this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (curPubSet) => { + return pubSet.mainPersonPub.id !== curPubSet.mainPersonPub.id + }) + // update overall reviewType + _.set(pubSet, 'reviewType', reviewType) + + // add to new lists + if (!this.personPubSetsByReviewType[reviewType]) this.personPubSetsByReviewType[reviewType] = [] + this.personPubSetsByReviewType[reviewType] = _.concat(this.personPubSetsByReviewType[reviewType], pubSet) + if (!this.filteredPersonPublicationsCombinedMatchesByReview[reviewType]) this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] = [] + this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] = _.concat(this.filteredPersonPublicationsCombinedMatchesByReview[reviewType], pubSet) + if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { + const currentPersonIndex = _.findIndex(this.people, (person) => { + return person.id === this.person.id + }) + this.personReviewedPubCounts[this.person.id] += 1 + this.people[currentPersonIndex].person_publication_count -= 1 + await this.changedPendingCounts(currentPersonIndex) + // this.people[currentPersonIndex].reviews_persons_publications_aggregate.aggregate.count = 1 + this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count -= 1 + } else if (this.selectedPersonTotal === 'Pending' && reviewType === 'pending') { + const currentPersonIndex = _.findIndex(this.people, (person) => { + return person.id === this.person.id + }) + this.personReviewedPubCounts[this.person.id] -= 1 + this.people[currentPersonIndex].person_publication_count += 1 + await this.changedPendingCounts(currentPersonIndex) + // this.people[currentPersonIndex].reviews_persons_publications_aggregate.aggregate.count += 1 + this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count += 1 + } + this.publicationsReloadPending = true console.log(`Added reviews: ${JSON.stringify(mutateResults, null, 2)}`) this.clearPublication() return mutateResults From 94505dae33be5393be055913ffd4fd23372502fb Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 10 Aug 2021 11:09:24 -0400 Subject: [PATCH 32/43] fix pub load bugs and flicker of sourcename chip --- client/src/pages/Index.vue | 83 +++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 60345a6b..258fc167 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -140,31 +140,31 @@ - + Move To: @@ -231,16 +231,16 @@ target="_blank" /> - - Title: {{ publication.title }} + + Title: {{ personPublication.publication.title }} Citation: {{ publicationCitation }} - + Journal Title: {{ publication.journal_title }} - + Journal Subjects: {{classification.name}} @@ -318,7 +318,7 @@ > - Total: {{ confidenceSet.value }} + Total: {{ (confidenceSet ? confidenceSet.value : 'unknown') }} @@ -459,6 +459,8 @@ export default { personPubSetsByReviewType: {}, publicationsGroupedByTitleByReview: {}, publicationsGroupedByDoiByReview: {}, + // this will contain the main publication used for display for pubsets + personPublicationsCombinedMatchesByReview: {}, filteredPersonPublicationsCombinedMatchesByReview: {}, // will be a personsPublication id mapped to an object of {titleKey. doiKey} personPublicationsKeys: {}, @@ -1221,6 +1223,7 @@ export default { async clearPublications () { this.publications = [] this.personPublicationsCombinedMatches = [] + this.personPublicationsCombinedMatchesByReview = {} this.filteredPersonPublicationsCombinedMatchesByReview = {} this.publicationsGroupedByTitleByReview = {} this.publicationsGroupedByDoiByReview = {} @@ -1339,6 +1342,12 @@ export default { return pubSet.reviewType }) + // now group main pubs from pubset into separate combined matches map for display to make faster (fixes flicker in chip color for source) + this.personPublicationsCombinedMatchesByReview = _.mapValues(this.personPubSetsByReviewType, (pubSets) => { + return _.map(pubSets, (pubSet) => { + return pubSet.mainPersonPub + }) + }) // console.log(`PersonPublication Sets by Review Type are: ${JSON.stringify(this.personPubSetsByReviewType, null, 2)}`) // console.log(`Publications grouped by title by review: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) @@ -1390,18 +1399,13 @@ export default { async filterPublications () { let filterOutCurrentPublication = false this.filteredPersonPublicationsCombinedMatchesByReview = _.mapValues( - this.personPubSetsByReviewType, - (personPubSets) => { - return _.filter(personPubSets, (pubSet) => { - const personPub = this.getMainPersonPubFromSet(pubSet) - // console.log(`Person pub on filter is: ${JSON.stringify(personPub, null, 2)}, person pub sets are: ${JSON.stringify(personPubSets, null, 2)}`) + this.personPublicationsCombinedMatchesByReview, + (personPubs) => { + return _.filter(personPubs, (personPub) => { let includePublication = personPub.publication.title.toLowerCase().includes(this.pubSearch.toLowerCase().trim()) if (includePublication) { // also check if confidence is to be filtered out - // console.log('checking if we should include publication') - // console.log(`confidence val is: ${this.getPublicationConfidence(item)}`) if (this.selectedPersonConfidence === '50%' && this.getPublicationConfidence(personPub) < 0.50) { - // console.log('trying to filter out publication') includePublication = false } } @@ -1455,24 +1459,20 @@ export default { // apply any sorting applied console.log('sorting', this.selectedPersonPubSort) if (this.selectedPersonPubSort === 'Title') { - this.personPublicationsCombinedMatches = _.sortBy(this.personPublicationsCombinedMatches, (personPubSet) => { - // console.log(`Placing in sort of personPubSet: ${JSON.stringify(personPubSet, null, 2)}`) - const personPub = this.getMainPersonPubFromSet(personPubSet) + this.personPublicationsCombinedMatches = _.sortBy(this.personPublicationsCombinedMatches, (personPub) => { return this.trimFirstArticles(personPub.publication.title) }) } else { // need to sort by confidence and then name, not guaranteed to be in order from what is returned from DB // first group items by count - const pubsByConf = _.groupBy(this.personPublicationsCombinedMatches, (pubSet) => { - const pub = this.getMainPersonPubFromSet(pubSet) + const pubsByConf = _.groupBy(this.personPublicationsCombinedMatches, (pub) => { return this.getPublicationConfidence(pub) }) // sort each person array by title for each conf - const pubsByConfByName = _.mapValues(pubsByConf, (pubSets) => { - return _.sortBy(pubSets, (pubSet) => { - const mainPersonPub = this.getMainPersonPubFromSet(pubSet) - return mainPersonPub.publication.title + const pubsByConfByName = _.mapValues(pubsByConf, (pubs) => { + return _.sortBy(pubs, (pub) => { + return pub.publication.title }) }) @@ -1548,9 +1548,13 @@ export default { // console.log(`Loaded Publication: ${JSON.stringify(this.publication)}`) // console.log(`Publication journal is: ${JSON.stringify(this.publication.journal_title, null, 2)}`) this.publicationCitation = this.getCitationApa(this.publication.csl_string) - this.publicationJournalClassifications = _.map(this.publication.journal.journals_classifications_aggregate.nodes, (node) => { - return node.classification - }) + if (this.publication.journal) { + this.publicationJournalClassifications = _.map(this.publication.journal.journals_classifications_aggregate.nodes, (node) => { + return node.classification + }) + } else { + this.publicationJournalClassifications = [] + } // console.log(`Found Journal Classifications: ${JSON.stringify(this.publicationJournalClassifications, null, 2)}`) try { const sanitizedDoi = sanitize(this.publication.doi, { replacement: '_' }) @@ -1611,8 +1615,11 @@ export default { this.personPubSetsByReviewType[this.reviewTypeFilter] = _.filter(this.personPubSetsByReviewType[this.reviewTypeFilter], (curPubSet) => { return pubSet.mainPersonPub.id !== curPubSet.mainPersonPub.id }) - this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter] = _.filter(this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (curPubSet) => { - return pubSet.mainPersonPub.id !== curPubSet.mainPersonPub.id + this.personPublicationsCombinedMatchesByReview[this.reviewTypeFilter] = _.filter(this.personPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (personPub) => { + return pubSet.mainPersonPub.id !== personPub.id + }) + this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter] = _.filter(this.filteredPersonPublicationsCombinedMatchesByReview[this.reviewTypeFilter], (curPub) => { + return pubSet.mainPersonPub.id !== curPub.id }) // update overall reviewType _.set(pubSet, 'reviewType', reviewType) @@ -1621,7 +1628,9 @@ export default { if (!this.personPubSetsByReviewType[reviewType]) this.personPubSetsByReviewType[reviewType] = [] this.personPubSetsByReviewType[reviewType] = _.concat(this.personPubSetsByReviewType[reviewType], pubSet) if (!this.filteredPersonPublicationsCombinedMatchesByReview[reviewType]) this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] = [] - this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] = _.concat(this.filteredPersonPublicationsCombinedMatchesByReview[reviewType], pubSet) + this.filteredPersonPublicationsCombinedMatchesByReview[reviewType] = _.concat(this.filteredPersonPublicationsCombinedMatchesByReview[reviewType], pubSet.mainPersonPub) + if (!this.personPublicationsCombinedMatchesByReview[reviewType]) this.personPublicationsCombinedMatchesByReview[reviewType] = [] + this.personPublicationsCombinedMatchesByReview[reviewType] = _.concat(this.personPublicationsCombinedMatchesByReview[reviewType], pubSet.mainPersonPub) if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { const currentPersonIndex = _.findIndex(this.people, (person) => { return person.id === this.person.id From 400756a5374eb4e6ff4ae2654ed203ab337589ff Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 12 Aug 2021 10:45:33 -0400 Subject: [PATCH 33/43] add grouping by pubsets to center review --- client/src/pages/CenterReview.vue | 599 ++++++++++++++++++------------ client/src/pages/Index.vue | 2 +- 2 files changed, 363 insertions(+), 238 deletions(-) diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index d08b14a7..3c60bee2 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -97,15 +97,15 @@ Title: {{ decode(item.publication.title) }} - {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByTitle[selectedInstitutionReviewState.toLowerCase()][item.publication.title] }} - {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByTitle[selectedInstitutionReviewState.toLowerCase()][item.publication.title] }} + {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByTitle[selectedInstitutionReviewState.toLowerCase()][getPublicationTitleKey(item.publication.title)] }} + {{selectedInstitutionReviewState}} Authors: {{ sortAuthorsByTitle[selectedInstitutionReviewState.toLowerCase()][getPublicationTitleKey(item.publication.title)] }} Person Pub Set ID Pointers + personPubSetPointer: {}, + // PersonPubSet Id -> Person Pub Id list (i.e., the set itself) + personPubSetsById: {}, + // the current index for personPubSets, will increment whenever adding a new one + personPubSetIdIndex: 0, + personPublicationsById: {}, + personPubSetsByReviewType: {}, + personPublicationsKeys: {}, publicationsGroupedByInstitutionReview: {}, personPublicationsCombinedMatches: [], personPublicationsCombinedMatchesByReview: {}, @@ -435,6 +446,8 @@ export default { filteredPersonPublicationsCombinedMatchesByOrgReview: {}, publicationsGroupedByTitleByInstitutionReview: {}, publicationsGroupedByTitleByOrgReview: {}, + publicationsGroupedByDoiByInstitutionReview: {}, + publicationsGroupedByDoiByOrgReview: {}, publicationsGroupedByTitle: {}, sortAuthorsByTitle: {}, // map of title's to the matched author to sort by (i.e., the matched author with the lowest matched position) institutions: [], @@ -567,6 +580,181 @@ export default { } }, methods: { + getPersonPublicationsInSet (pubSet) { + return _.map(pubSet.personPublicationIds, (pubId) => { + return this.getPersonPublicationById(pubId) + }) + }, + getMainPersonPubFromSet (pubSet) { + return pubSet.mainPersonPub + }, + // will link all personPubs in this list together + linkPersonPubs (personPubList, reviewType) { + // link all person pubs together in this list + const totalPubs = personPubList.length + _.each(personPubList, (personPub, index) => { + // at last item do nothing + try { + if (index === 0 && totalPubs === 1) { + // console.log(`Linking index: ${index}, only one pub in set, personpub: ${JSON.stringify(personPub['id'], null, 2)}`) + // will start a new personpub set list if not already in one + // console.log(`Passing person pub id to start a new set: ${personPub['id']}`) + this.startPersonPubSet(personPub['id'], reviewType) + } else if (index !== (totalPubs - 1)) { + // console.log(`Linking index: ${index} to ${index + 1} of ${totalPubs}), personpub: ${JSON.stringify(personPub['id'], null, 2)}`) + const nextPersonPub = _.nth(personPubList, (index + 1)) + this.linkPersonPubPair(personPub.id, nextPersonPub.id, reviewType) + } + } catch (error) { + console.log(`Warning, error on linking publications: ${error}`) + throw error + } + }) + }, + // this method will link person pubs by putting them in a person pub set + // together. If neither are already in a person pub set, they will be grouped together in a new set + // If only one has a set, the other will be added to that set + // If both are currently within a set, it will merge the two sets together + linkPersonPubPair (personPub1Id, personPub2Id, reviewType) { + const notInPersonPub1SetId = this.notInPersonPubSet(personPub1Id) + const notInPersonPub2SetId = this.notInPersonPubSet(personPub2Id) + const personPubSet1Id = this.getPersonPubSetId(personPub1Id) + const personPubSet2Id = this.getPersonPubSetId(personPub2Id) + console.log(`Linking person pub id 1: ${personPub1Id} to person pub id 2: ${personPub2Id}`) + if (notInPersonPub1SetId && notInPersonPub2SetId) { + // neither one is in a set yet and just add to set list + // console.log(`Starting new set for personPubId1: ${personPub1Id}`) + const newSetId = this.startPersonPubSet(personPub1Id, reviewType) + // console.log(`Created personPubset1: ${newSetId} and added person pub1: ${personPub1Id}`) + // console.log(`Adding personPubId2: ${personPub2Id} to set1id: ${newSetId}`) + this.addPersonPubToSet(newSetId, personPub2Id, reviewType) + } else if (notInPersonPub2SetId) { + // console.log(`Adding personPubId2: ${personPub2Id} to set1id: ${personPubSet1Id}`) + this.addPersonPubToSet(personPubSet1Id, personPub2Id, reviewType) + } else if (notInPersonPub1SetId) { + // console.log(`Adding personPubId1: ${personPub1Id} to set2id: ${personPubSet2Id}`) + this.addPersonPubToSet(personPubSet2Id, personPub1Id, reviewType) + } else { + // they are both in existing sets and need to merge them + // do nothing if they have the same pubsetid as they are already + // in the same set + // console.log(`Merging sets for personPubId1: ${personPub1Id} and personPubId2: ${personPub2Id}, set2Id->set1Id: (${personPubSet2Id}->${personPubSet1Id})`) + this.mergePersonPubSets(personPubSet1Id, personPubSet2Id, reviewType) + } + }, + mergePersonPubSets (set1Id, set2Id, reviewType) { + // do nothing if they are the same set id + // console.log(`Merging person pub sets set1: ${set1Id} set2: ${set2Id}`) + if (set1Id !== set2Id) { + // add items from set2 into set1 if not already there, assumes everything is up to date with pointers + const set1 = this.getPersonPubSet(set1Id) + const set2 = this.getPersonPubSet(set2Id) + if (set1.reviewType !== reviewType || set2.reviewType !== reviewType) { + const error = `Error: Mismatch in reviewType for sets to be merged. Expected: ${reviewType}, found set 1: ${set1.reviewType} set 2: ${set2.reviewType}` + console.log(error) + throw error + } + const set2List = set2.personPublicationIds + _.each(set2List, (personPubId) => { + this.addPersonPubToSet(set1Id, personPubId, reviewType) + }) + // destroy the set2List + this.removePersonPubSet(set2Id) + } + }, + removePersonPubSet (setId) { + // only works if personPubs in this set already pointing to another one, else throw error + // do nothing if set already gone + const set = this.getPersonPubSet(setId) + if (set) { + _.each(set, (personPubId) => { + if (setId && this.getPersonPubSetId(personPubId) === setId) { + const error = `Cannot remove person Pub Set (on merge), personPubId: ${personPubId} not in any other set` + console.log(error) + throw error + } + }) + // if we get this far no errors encountered, and all person pubs are now in another set + // go ahead and delete it + _.unset(this.personPubSetsById, setId) + } + }, + addPersonPubToSet (setId, personPubId, reviewType) { + // proceed if set exists + const set = this.getPersonPubSet(setId) + if (set) { + // do nothing if already in the set + if (this.getPersonPubSetId(personPubId) !== setId) { + if (set.reviewType !== reviewType) { + const error = `Failed to add person pub to set with mismatched review types. Expected ${reviewType}, found: ${set.reviewType}` + console.log(error) + throw error + } + const addPub = this.getPersonPublicationById(personPubId) + this.personPubSetsById[setId].personPublicationIds = _.concat(this.personPubSetsById[setId].personPublicationIds, personPubId) + this.personPubSetsById[setId].personPublications = _.concat(this.personPubSetsById[setId].personPublications, addPub) + this.personPubSetPointer[personPubId] = setId + // console.log(`After add personpub: ${personPubId} to setid: ${setId} pubset pointers are: ${JSON.stringify(this.personPubSetPointer, null, 2)}`) + const mainPersonPub = this.getPersonPublicationById(set.mainPersonPubId) + if (!set.mainPersonPubId || this.getPublicationConfidence(mainPersonPub) < this.getPublicationConfidence(addPub)) { + _.set(set, 'mainPersonPub', addPub) + _.set(set, 'mainPersonPubId', addPub.id) + } + } + } else { + const error = `Failed to add personPub with id: ${personPubId} to set id: ${setId}, personPubSet does not exist` + console.log(error) + throw error + } + }, + notInPersonPubSet (personPubId) { + // true if already in a set + return !this.personPubSetPointer[personPubId] + }, + getPersonPubSet (setId) { + return this.personPubSetsById[setId] + }, + // this method is not currently thread-safe + // creates a new person pub set if one does not already exist for the given + // person Pub Id + startPersonPubSet (personPubId, reviewType) { + if (this.notInPersonPubSet(personPubId)) { + // console.log(`Creating person pub set for pub id: ${personPubId}`) + const personPubSetId = this.getNextPersonPubSetId() + this.personPubSetPointer[personPubId] = personPubSetId + const personPub = this.getPersonPublicationById(personPubId) + this.personPubSetsById[personPubSetId] = { + personPublicationIds: [personPubId], + personPublications: [personPub], + mainPersonPubId: personPubId, + mainPersonPub: personPub, + reviewType: reviewType + } + return personPubSetId + } else { + const currentSetId = this.getPersonPubSetId(personPubId) + const currentSet = this.getPersonPubSet(currentSetId) + if (currentSet.reviewType !== reviewType) { + const error = `Error: Mismatch on review type for person Pub set for personPub id: ${personPubId}, expected review type: ${reviewType} and found review type: ${currentSet.reviewType}` + console.log(error) + throw error + } else { + return this.getPersonPubSetId(personPubId) + } + } + }, + getPersonPublicationById (personPubId) { + return this.personPublicationsById[personPubId] + }, + // returns a person Pub set if it exists for that personPub, else returns undefined + getPersonPubSetId (personPubId) { + return this.personPubSetPointer[personPubId] + }, + // this method is not currently thread-safe + getNextPersonPubSetId () { + this.personPubSetIdIndex += 1 + return this.personPubSetIdIndex + }, drawerClick (e) { // if in "mini" state and user // click on drawer, we switch it to "normal" mode @@ -674,26 +862,11 @@ export default { getPublicationsGroupedByTitleByOrgReviewCount (reviewType) { return this.filteredPersonPublicationsCombinedMatchesByOrgReview[reviewType] ? this.filteredPersonPublicationsCombinedMatchesByOrgReview[reviewType].length : 0 }, - getInstitutionReviewedAuthors (title, reviewType) { - const personPubs = this.publicationsGroupedByTitleByInstitutionReview[reviewType][title] - // console.log(`Person pubs on get author review: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) - const reviewedAuthors = {} - _.each(personPubs, (personPub) => { - const reviewedAuthor = this.getReviewedAuthor(personPub) - if (!reviewedAuthors[reviewedAuthor.id]) { - reviewedAuthors[reviewedAuthor.id] = reviewedAuthor - } - }) - if (reviewedAuthors.length > 1) { - console.log(`Reviewed authors are: ${JSON.stringify(reviewedAuthors, null, 2)}`) - } - return _.values(reviewedAuthors) - }, - getTitlePersonPublicationsByReview (title) { + getTitlePersonPublicationsByReview (titleKey) { const personPubsByReview = {} _.each(_.keys(this.publicationsGroupedByTitleByInstitutionReview), (reviewType) => { - if (this.publicationsGroupedByTitleByInstitutionReview[reviewType][title]) { - const pubsGroupedByPersonId = _.groupBy(this.publicationsGroupedByTitleByInstitutionReview[reviewType][title], (personPub) => { + if (this.publicationsGroupedByTitleByInstitutionReview[reviewType][titleKey]) { + const pubsGroupedByPersonId = _.groupBy(this.publicationsGroupedByTitleByInstitutionReview[reviewType][titleKey], (personPub) => { return personPub.person_id }) personPubsByReview[reviewType] = _.map(_.keys(pubsGroupedByPersonId), (personId) => { @@ -902,18 +1075,29 @@ export default { const personResult = await this.$apollo.query(readPersons()) this.people = personResult.data.persons }, + removeSpaces (value) { + if (_.isString(value)) { + return _.clone(value).replace(/\s/g, '') + } else { + return value + } + }, // replace diacritics with alphabetic character equivalents - normalizeString (value) { + normalizeString (value, lowerCase, removeSpaces) { if (_.isString(value)) { - const newValue = _.clone(value) - const norm1 = newValue + let newValue = _.clone(value) .normalize('NFD') - .replace(/[\u0300-\u036f]/g, '') - // the u0027 also normalizes the curly apostrophe to the straight one - const norm2 = norm1.replace(/[\u2019]/g, '\u0027') - // remove periods and other remaining special characters - const norm3 = norm2.replace(/[&\\#,+()$~%.'":*?<>{}!]/g, '') - return norm3 + .replace(/[\u0300-\u036f]/g, '') // Remove diacritics + .replace(/[\u2019]/g, '\u0027') // the u0027 also normalizes the curly apostrophe to the straight one + .replace(/[&\\#,+()$~%.'":*?<>{}!-]/g, '') // remove periods and other remaining special characters + if (lowerCase) { + newValue = _.lowerCase(newValue) + } + if (removeSpaces) { + return this.removeSpaces(newValue) + } else { + return newValue + } } else { return value } @@ -922,7 +1106,7 @@ export default { normalizeObjectProperties (object, properties) { const newObject = _.clone(object) _.each(properties, (property) => { - newObject[property] = this.normalizeString(newObject[property]) + newObject[property] = this.normalizeString(newObject[property], false, false) }) return newObject }, @@ -932,7 +1116,7 @@ export default { return this.normalizeObjectProperties(name, [lastKey]) }) // normalize last name checking against as well - const testLast = this.normalizeString(last) + const testLast = this.normalizeString(last, false, false) // console.log(`After diacritic switch ${JSON.stringify(nameMap, null, 2)} converted to: ${JSON.stringify(testNameMap, null, 2)}`) // const lastFuzzy = new VueFuse(testNameMap, { // caseSensitive: false, @@ -1106,16 +1290,28 @@ export default { return this.getPubCSVResultObject(personPub) }) }, + getPublicationTitleKey (title) { + // normalize the string and remove characters like dashes as well + return this.normalizeString(title, true, true) + }, + getPublicationDoiKey (publication) { + if (!publication.doi) { + return `${publication.source_name}_${publication.source_id}` + } else { + return publication.doi + } + }, getPubCSVResultObject (personPublication) { - const citation = (this.citationsByTitle[_.toLower(personPublication.publication.title)] ? this.citationsByTitle[_.toLower(personPublication.publication.title)] : undefined) + const titleKey = this.getPublicationTitleKey(personPublication.publication.title) + const citation = (this.citationsByTitle[titleKey] ? this.citationsByTitle[titleKey] : undefined) return { - authors: this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][personPublication.publication.title], + authors: this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][titleKey], title: personPublication.publication.title.replace(/\n/g, ' '), doi: this.getCSVHyperLinkString(personPublication.publication.doi, this.getDoiUrl(personPublication.publication.doi)), journal: (personPublication.publication.journal_title) ? personPublication.publication.journal_title : '', year: personPublication.publication.year, - source_names: JSON.stringify(_.map(this.getSortedPersonPublicationsBySourceName(this.publicationsGroupedByTitle[personPublication.publication.title]), (pub) => { return pub.publication.source_name })), - sources: this.getSourceUriString(this.getSortedPersonPublicationsBySourceName(this.publicationsGroupedByTitle[personPublication.publication.title])), + source_names: JSON.stringify(_.map(this.getSortedPersonPublicationsBySourceName(this.getPersonPubSet(this.getPersonPubSetId(personPublication.id)).personPublications), (pub) => { return pub.publication.source_name })), + sources: this.getSourceUriString(this.getSortedPersonPublicationsBySourceName(this.getPersonPubSet(this.getPersonPubSetId(personPublication.id)).personPublications)), abstract: personPublication.publication.abstract, citation: citation } @@ -1125,9 +1321,6 @@ export default { }, async loadPersonPublicationsCombinedMatches () { console.log(`Start group by publications ${moment().format('HH:mm:ss:SSS')}`) - this.publicationsGroupedByTitle = _.groupBy(this.publications, (personPub) => { - return `${personPub.publication.title}` - }) // this.fundersByDoi = {} // this.pubMedFundersByDoi = {} @@ -1135,160 +1328,120 @@ export default { // this.uniqueFunders = {} this.filteredPersonPubCounts = {} // group by institution (i.e., ND author) review and then by doi - let pubsByTitle = {} - this.publicationsGroupedByInstitutionReview = _.groupBy(this.publications, function (personPub) { + // let pubsByTitle = {} + const thisVue = this + this.publicationsGroupedByInstitutionReview = _.groupBy(thisVue.publications, function (personPub) { let reviewType = 'pending' - const title = personPub.publication.title + if (!thisVue.personPublicationsById) thisVue.personPublicationsById = {} + thisVue.personPublicationsById[personPub.id] = personPub + // const title = personPub.publication.title // if (doi === '10.1101/gad.307116.117') { // console.log(`Person pub for doi 10.1101/gad.307116.117 is: ${JSON.stringify(personPub, null, 2)}`) // } if (personPub.reviews && personPub.reviews.length > 0) { reviewType = personPub.reviews[0].review_type } - if (!pubsByTitle[reviewType]) { - pubsByTitle[reviewType] = {} - } - if (!pubsByTitle[reviewType][title]) { - pubsByTitle[reviewType][title] = [] - } - pubsByTitle[reviewType][title].push(personPub) return reviewType }) // console.log(`Pubs by doi are: ${JSON.stringify(pubsByTitle, null, 2)}`) // put in pubs grouped by doi for each review status - this.publicationsGroupedByTitleByInstitutionReview = pubsByTitle - console.log(`Finish group by publications ${moment().format('HH:mm:ss:SSS')}`) - // initialize the pub author matches - this.matchedPublicationAuthorsByTitle = _.mapValues(this.authorsByTitle, (cslAuthors, title) => { - // console.log(`DOIs that are matched: ${JSON.stringify(_.keys(this.publicationsGroupedByTitleByInstitutionReview), null, 2)}`) - return this.getMatchedCslAuthors(cslAuthors, this.publicationsGroupedByTitleByInstitutionReview['accepted'][title], true) - }) - this.sortAuthorsByTitle = {} - this.sortAuthorsByTitle['accepted'] = _.mapValues(this.matchedPublicationAuthorsByTitle, (matchedAuthors) => { - this.updateFilteredPersonPubCounts('accepted', matchedAuthors) - return this.getAuthorsString(matchedAuthors) - }) - // this.sortAuthorsByTitle['rejected'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['rejected'], (personPubs, title) => { - // return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'rejected')) - // }) - // this.sortAuthorsByTitle['unsure'] = _.mapValues(this.publicationsGroupedByTitleByInstitutionReview['unsure'], (personPubs, title) => { - // return this.getAuthorsString(this.getInstitutionReviewedAuthors(title, 'unsure')) - // }) - - // console.log(`Matched pub authors by doi: ${JSON.stringify(this.matchedPublicationAuthorsByTitle, null, 2)}`) + // this.publicationsGroupedByTitleByInstitutionReview = pubsByTitle - const titlesPresent = {} - this.personPublicationsCombinedMatchesByReview = {} - // console.log(`Person pubs grouped by title are: ${JSON.stringify(this.publicationsGroupedByTitleByInstitutionReview, null, 2)}`) - // grab one with highest confidence to display and grab others via title later when changing status - // instead of random order add them sequentially to not have duplicates in rejected, - // unsure if in accepted and then not in rejected if unsure and not in accepted - this.personPublicationsCombinedMatchesByReview['accepted'] = await _.map(_.keys(this.publicationsGroupedByTitleByInstitutionReview['accepted']), (title) => { - // get match with highest confidence level and use that one - const personPubs = this.publicationsGroupedByTitleByInstitutionReview['accepted'][title] - let currentPersonPub - _.each(personPubs, (personPub, index) => { - if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { - currentPersonPub = personPub - // if (currentPersonPub.publication.crossref_funders && currentPersonPub.publication.crossref_funders.length > 0) { - // this.fundersByDoi[doi] = (currentPersonPub.publication.crossref_funders) ? currentPersonPub.publication.crossref_funders : [] - // this.combinedFundersByDoi[doi] = (currentPersonPub.publication.crossref_funders) ? currentPersonPub.publication.crossref_funders : [] - // _.each(this.fundersByDoi[doi], (funder) => { - // console.log(`Getting name from funder: ${JSON.stringify(funder, null, 2)}`) - // const funderName = funder['name'] - // console.log(`Got name ${funderName} from funder: ${JSON.stringify(funder, null, 2)}`) - // if (funderName) { - // if (!this.uniqueFunders[funderName]) { - // this.uniqueFunders[funderName] = [] - // } - // if (this.uniqueFunders[funderName]) { - // console.log(`Pushing funder name: ${funderName}`) - // this.uniqueFunders[funderName].push(funder) - // } - // } - // console.log(`doi: ${doi} crossref funders: ${JSON.stringify(funder, null, 2)}`) - // }) - // } + // start add code for pubsets + // map both by shared title and by shared doi and merged lists together later + let publicationTitlesByReviewType = {} + let publicationDoisByReviewType = {} + // put in pubs grouped by doi for each review status + _.each(this.reviewStates, (reviewType) => { + const publications = this.publicationsGroupedByInstitutionReview[reviewType] - // if (currentPersonPub.publication.pubmed_funders && currentPersonPub.publication.pubmed_funders.length > 0) { - // this.pubMedFundersByDoi[doi] = (currentPersonPub.publication.pubmed_funders) ? currentPersonPub.publication.pubmed_funders : [] - // this.combinedFundersByDoi[doi] = (currentPersonPub.publication.pubmed_funders) ? currentPersonPub.publication.pubmed_funders : [] - // _.each(this.pubMedFundersByDoi[doi], (funder) => { - // console.log(`Getting name from funder: ${JSON.stringify(funder, null, 2)}`) - // const funderName = funder['funder'] - // console.log(`Got name ${funderName} from funder: ${JSON.stringify(funder, null, 2)}`) - // if (funderName) { - // if (!this.uniqueFunders[funderName]) { - // this.uniqueFunders[funderName] = [] - // } - // if (this.uniqueFunders[funderName]) { - // console.log(`Pushing funder name: ${funderName}`) - // this.uniqueFunders[funderName].push(funder) - // } - // } - // console.log(`doi: ${doi} pubmed funders: ${JSON.stringify(funder, null, 2)}`) - // }) - // } + // seed the personPublication keys + _.each(publications, (personPub) => { + this.personPublicationsKeys[personPub.id] = { + titleKey: this.getPublicationTitleKey(personPub.publication.title), + doiKey: this.getPublicationDoiKey(personPub.publication) } }) - titlesPresent[title] = true - return currentPersonPub - }) + this.publicationsGroupedByTitleByInstitutionReview[reviewType] = _.groupBy(publications, (personPub) => { + // let title = personPub.publication.title + const titleKey = this.personPublicationsKeys[personPub.id].titleKey + // console.log(`Title: '${title} title key: ${titleKey}'`) + if (!publicationTitlesByReviewType[titleKey]) { + publicationTitlesByReviewType[titleKey] = {} + } + if (!publicationTitlesByReviewType[titleKey][reviewType]) { + publicationTitlesByReviewType[titleKey][reviewType] = reviewType + } + // publicationTitlesByReviewType[titleKey][reviewType].push(personPub) + return `${titleKey}` + }) - this.personPublicationsCombinedMatchesByReview['unsure'] = {} - await pMap(_.keys(this.publicationsGroupedByTitleByInstitutionReview['unsure']), (title) => { - // get match with highest confidence level and use that one - if (!titlesPresent[title]) { - const personPubs = this.publicationsGroupedByTitleByInstitutionReview['unsure'][title] - let currentPersonPub - _.each(personPubs, (personPub, index) => { - if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { - currentPersonPub = personPub - } - }) + this.publicationsGroupedByDoiByInstitutionReview[reviewType] = _.groupBy(publications, (personPub) => { + // let doi = personPub.publication.doi + const doiKey = this.personPublicationsKeys[personPub.id].doiKey + // console.log(`Doi: '${doi} doi key: ${doiKey}'`) + if (!publicationDoisByReviewType[doiKey]) { + publicationDoisByReviewType[doiKey] = {} + } + if (!publicationDoisByReviewType[doiKey][reviewType]) { + publicationDoisByReviewType[doiKey][reviewType] = reviewType + } + // publicationDoisByReviewType[doiKey][reviewType].push(personPub) + return `${doiKey}` + }) - const unsureAuthors = this.getInstitutionReviewedAuthors(title, 'unsure') - this.updateFilteredPersonPubCounts('unsure', unsureAuthors) - titlesPresent[title] = true - this.personPublicationsCombinedMatchesByReview['unsure'][title] = currentPersonPub - } - }, { concurrency: 1 }) + // console.log(`Person pubs grouped by Title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) + // grab one with highest confidence to display and grab others via title later when changing status - this.personPublicationsCombinedMatchesByReview['rejected'] = {} - await pMap(_.keys(this.publicationsGroupedByTitleByInstitutionReview['rejected']), (title) => { - // get match with highest confidence level and use that one - if (!titlesPresent[title]) { - const personPubs = this.publicationsGroupedByTitleByInstitutionReview['rejected'][title] - let currentPersonPub - _.each(personPubs, (personPub, index) => { - if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { - currentPersonPub = personPub - } - }) + // this.personPublicationsCombinedMatchesByReview[reviewType] = {} + console.log(`Create publication graph...`) + // keep a map of personPubId to set id in order to find the set that something should be added to if found as same pub + // merge personPubs together by title and then doi + _.each(_.keys(this.publicationsGroupedByTitleByInstitutionReview[reviewType]), (titleKey) => { + this.linkPersonPubs(this.publicationsGroupedByTitleByInstitutionReview[reviewType][titleKey], reviewType) + }) - const rejectedAuthors = this.getInstitutionReviewedAuthors(title, 'rejected') - this.updateFilteredPersonPubCounts('rejected', rejectedAuthors) - titlesPresent[title] = true - this.personPublicationsCombinedMatchesByReview['rejected'][title] = currentPersonPub - } - }, { concurrency: 1 }) + // now link together if same doi (if already found above will add to existing set) + _.each(_.keys(this.publicationsGroupedByDoiByInstitutionReview[reviewType]), (doiKey) => { + this.linkPersonPubs(this.publicationsGroupedByDoiByInstitutionReview[reviewType][doiKey], reviewType) + }) + console.log(`Finished publication graph.`) - this.personPublicationsCombinedMatchesByReview['pending'] = {} - await pMap(_.keys(this.publicationsGroupedByTitleByInstitutionReview['pending']), (title) => { // get match with highest confidence level and use that one - if (!titlesPresent[title]) { - const personPubs = this.publicationsGroupedByTitleByInstitutionReview['pending'][title] - let currentPersonPub - _.each(personPubs, (personPub, index) => { - if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { - currentPersonPub = personPub - } - }) + // let currentPersonPub + // _.each(personPubs, (personPub, index) => { + // if (!currentPersonPub || this.getPublicationConfidence(currentPersonPub) < this.getPublicationConfidence(personPub)) { + // currentPersonPub = personPub + // } + // }) + // return currentPersonPub + // }) + }) - this.personPublicationsCombinedMatchesByReview['pending'][title] = currentPersonPub - } - }, { concurrency: 1 }) + // group pub sets by review type + this.personPubSetsByReviewType = _.groupBy(_.values(this.personPubSetsById), (pubSet) => { + return pubSet.reviewType + }) + + // now group main pubs from pubset into separate combined matches map for display to make faster (fixes flicker in chip color for source) + this.personPublicationsCombinedMatchesByReview = _.mapValues(this.personPubSetsByReviewType, (pubSets) => { + return _.map(pubSets, (pubSet) => { + return pubSet.mainPersonPub + }) + }) + // end add code for pubsets + console.log(`Finish group by publications ${moment().format('HH:mm:ss:SSS')}`) + // initialize the pub author matches + this.matchedPublicationAuthorsByTitle = _.mapValues(this.authorsByTitle, (cslAuthors, titleKey) => { + // console.log(`DOIs that are matched: ${JSON.stringify(_.keys(this.publicationsGroupedByTitleByInstitutionReview), null, 2)}`) + return this.getMatchedCslAuthors(cslAuthors, this.publicationsGroupedByTitleByInstitutionReview['accepted'][titleKey], true) + }) + this.sortAuthorsByTitle = {} + this.sortAuthorsByTitle['accepted'] = _.mapValues(this.matchedPublicationAuthorsByTitle, (matchedAuthors) => { + this.updateFilteredPersonPubCounts('accepted', matchedAuthors) + return this.getAuthorsString(matchedAuthors) + }) // now group by org review according to the selected institution review state if (!this.selectedInstitutionReviewState) { @@ -1296,6 +1449,9 @@ export default { } this.personPublicationsCombinedMatchesByOrgReview = _.groupBy(this.personPublicationsCombinedMatchesByReview[this.selectedInstitutionReviewState.toLowerCase()], function (pub) { if (pub.org_reviews && pub.org_reviews.length > 0) { + if (pub.org_reviews[0].person_id === 94) { + console.log(`Found review org review for ${pub.org_reviews[0].person_id}, review: ${JSON.stringify(pub.org_reviews, null, 2)}`) + } return pub.org_reviews[0].review_type } else { return 'pending' @@ -1309,15 +1465,6 @@ export default { } }) - // put in pubs grouped by title for each review status for both ND author review and center review status - _.each(this.reviewStates, (reviewType) => { - const publications = this.personPublicationsCombinedMatchesByOrgReview[reviewType] - // first grab relevant person pubs from global list based titles in this list - this.publicationsGroupedByTitleByOrgReview[reviewType] = _.groupBy(publications, (personPub) => { - return `${personPub.publication.title}` - }) - }) - this.loadPersonsWithFilter() // initialize the list in view @@ -1352,7 +1499,8 @@ export default { this.personPublicationsCombinedMatchesByOrgReview, (personPublications) => { return _.filter(personPublications, (item) => { - const authorString = (this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][item.publication.title]) ? this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][item.publication.title] : '' + const titleKey = this.getPublicationTitleKey(item.publication.title) + const authorString = (this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][titleKey]) ? this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][titleKey] : '' let includedInSelectedAuthors = true if (this.selectedCenterAuthor !== 'All') { // assumes the name value in the list of the same form as the author string @@ -1419,30 +1567,9 @@ export default { } else if (this.selectedCenterPubSort === 'Authors') { console.log('trying to sort by author') this.personPublicationsCombinedMatches = _.sortBy(this.personPublicationsCombinedMatches, (personPub) => { - return this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][personPub.publication.title] - }) - } else if (this.selectedCenterPubSort === 'Source') { - // need to sort by confidence and then name, not guaranteed to be in order from what is returned from DB - // first group items by count - const groupedPubs = _.groupBy(this.personPublicationsCombinedMatches, (pub) => { - return pub.publication.source_name - }) - - // sort each person array by title for each conf - const groupedPubsByTitle = _.mapValues(groupedPubs, (pubs) => { - return _.sortBy(pubs, ['title']) + const titleKey = this.getPublicationTitleKey(personPub.publication.title) + return this.sortAuthorsByTitle[this.selectedInstitutionReviewState.toLowerCase()][titleKey] }) - - // get array of pub values (i.e., keys) sorted in reverse - const sortedKeys = _.sortBy(_.keys(groupedPubsByTitle), (key) => { return key }) - - // now push values into array in desc order of count and flatten - let sortedPubs = [] - _.each(sortedKeys, (key) => { - sortedPubs.push(groupedPubsByTitle[key]) - }) - - this.personPublicationsCombinedMatches = _.flatten(sortedPubs) } else { // need to sort by confidence and then name, not guaranteed to be in order from what is returned from DB // first group items by count @@ -1569,7 +1696,7 @@ export default { return pub }) pubsWithAuthorsByTitle = _.groupBy(authorsPubs, (publication) => { - return publication.title + return this.getPublicationTitleKey(publication.title) }) } @@ -1578,6 +1705,7 @@ export default { this.authorsByTitle = _.mapValues(pubsWithAuthorsByTitle, (publication) => { return (publication[0].authors) ? publication[0].authors : [] }) + // console.log(`Authors by title are: ${JSON.stringify(this.authorsByTitle, null, 2)}`) await this.loadPersonPublicationsCombinedMatches() this.publicationsLoaded = true } @@ -1614,7 +1742,7 @@ export default { // console.log(`Starting group by title for citations batch ${index} ${moment().format('HH:mm:ss:SSS')}`) batchesPubsCSLByTitle.push(_.groupBy(pubsCSLResult.data.publications, (publication) => { - return _.toLower(publication.title) + return this.getPublicationTitleKey(publication.title) })) // console.log(`Finished group by title for citations batch ${index} ${moment().format('HH:mm:ss:SSS')}`) }, { concurrency: 1 }) @@ -1623,9 +1751,9 @@ export default { console.log(`Starting generate citations ${moment().format('HH:mm:ss:SSS')}`) await pMap(batchesPubsCSLByTitle, async (pubsCSLByTitle) => { console.log(`Generating citations for ${_.keys(pubsCSLByTitle).length} publications`) - await pMap(_.keys(pubsCSLByTitle), async (title) => { - if (!this.citationsByTitle[title]) { - this.citationsByTitle[title] = this.getCitationApa(pubsCSLByTitle[title][0].csl_string) + await pMap(_.keys(pubsCSLByTitle), async (titleKey) => { + if (!this.citationsByTitle[titleKey]) { + this.citationsByTitle[titleKey] = this.getCitationApa(pubsCSLByTitle[titleKey][0].csl_string) } }, { concurrency: 1 }) }, { concurrency: 1 }) @@ -1653,7 +1781,7 @@ export default { async loadPublication (personPublication) { this.clearPublication() this.personPublication = personPublication - const personPublicationsByReview = await this.getTitlePersonPublicationsByReview(personPublication.publication.title) + const personPublicationsByReview = await this.getTitlePersonPublicationsByReview(this.getPublicationTitleKey(personPublication.publication.title)) const reviewedAuthors = [] this.acceptedAuthors = _.map(personPublicationsByReview['accepted'], (personPub) => { const reviewedAuthor = this.getReviewedAuthor(personPub) @@ -1686,7 +1814,7 @@ export default { _.set(this.publication, 'uniqueAwards', _.mapKeys(this.publication.awards, (award) => { return award.funder_award_identifier })) - console.log(`Loaded Publication: ${JSON.stringify(this.publication)}`) + // console.log(`Loaded Publication: ${JSON.stringify(this.publication)}`) this.publicationCitation = this.getCitationApa(this.publication.csl_string) if (this.publication.journal && this.publication.journal.journals_classifications_aggregate) { this.publicationJournalClassifications = _.map(this.publication.journal.journals_classifications_aggregate.nodes, (node) => { @@ -1725,8 +1853,8 @@ export default { return null } // add the review for personPublications with the same title in the list - const personPubs = this.publicationsGroupedByTitle[personPublication.publication.title] - + const pubSet = this.getPersonPubSet(this.getPersonPubSetId(personPublication.id)) + const personPubs = pubSet.personPublications try { let mutateResults = [] await _.each(personPubs, async (personPub) => { @@ -1743,34 +1871,31 @@ export default { if (mutateResult && personPub.id === personPublication.id) { this.$refs[`personPub${index}`].hide() Vue.delete(this.personPublicationsCombinedMatches, index) - // transfer from one review queue to the next primarily for counts, other sorting will shake out on reload when clicking the tab - // remove from current lists - _.unset(this.publicationsGroupedByTitleByOrgReview[this.reviewTypeFilter], personPublication.publication.title) - _.remove(this.personPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter], (pub) => { - return pub.id === personPub.id - }) - _.remove(this.filteredPersonPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter], (pub) => { - return pub.id === personPub.id - }) - // add to new lists - this.publicationsGroupedByTitleByOrgReview[reviewType][personPublication.publication.title] = personPubs - this.personPublicationsCombinedMatchesByOrgReview[reviewType].push(personPub) - this.filteredPersonPublicationsCombinedMatchesByOrgReview[reviewType].push(personPub) - if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { - const currentPersonIndex = _.findIndex(this.people, (person) => { - return person.id === this.person.id - }) - this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count -= 1 - } else if (this.selectedPersonTotal === 'Pending' && reviewType === 'pending') { - const currentPersonIndex = _.findIndex(this.people, (person) => { - return person.id === this.person.id - }) - this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count += 1 - } } mutateResults.push(mutateResult) this.publicationsReloadPending = true }) + // remove set from related lists + this.personPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter] = _.filter(this.personPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter], (personPub) => { + return pubSet.mainPersonPub.id !== personPub.id + }) + this.filteredPersonPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter] = _.filter(this.filteredPersonPublicationsCombinedMatchesByOrgReview[this.reviewTypeFilter], (curPub) => { + return pubSet.mainPersonPub.id !== curPub.id + }) + // add to new lists + this.personPublicationsCombinedMatchesByOrgReview[reviewType].push(pubSet.mainPersonPub) + this.filteredPersonPublicationsCombinedMatchesByOrgReview[reviewType].push(pubSet.mainPersonPub) + if (this.reviewTypeFilter === 'pending' && this.selectedPersonTotal === 'Pending') { + const currentPersonIndex = _.findIndex(this.people, (person) => { + return person.id === this.person.id + }) + this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count -= 1 + } else if (this.selectedPersonTotal === 'Pending' && reviewType === 'pending') { + const currentPersonIndex = _.findIndex(this.people, (person) => { + return person.id === this.person.id + }) + this.people[currentPersonIndex].persons_publications_metadata_aggregate.aggregate.count += 1 + } console.log(`Added reviews: ${JSON.stringify(mutateResults, null, 2)}`) this.clearPublication() return mutateResults diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 258fc167..828f8343 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -1279,7 +1279,7 @@ export default { _.each(publications, (personPub) => { this.personPublicationsKeys[personPub.id] = { titleKey: this.getPublicationTitleKey(personPub.publication.title), - doiKey: this.getPublicationTitleKey(personPub.publication.doi) + doiKey: this.getPublicationDoiKey(personPub.publication) } }) this.publicationsGroupedByTitleByReview[reviewType] = _.groupBy(publications, (personPub) => { From 39878297174ce156929139bd9e8f26da88c00e25 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 12 Aug 2021 10:48:35 -0400 Subject: [PATCH 34/43] remove temp log message --- client/src/pages/CenterReview.vue | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index 3c60bee2..2f0b08f0 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -1449,9 +1449,6 @@ export default { } this.personPublicationsCombinedMatchesByOrgReview = _.groupBy(this.personPublicationsCombinedMatchesByReview[this.selectedInstitutionReviewState.toLowerCase()], function (pub) { if (pub.org_reviews && pub.org_reviews.length > 0) { - if (pub.org_reviews[0].person_id === 94) { - console.log(`Found review org review for ${pub.org_reviews[0].person_id}, review: ${JSON.stringify(pub.org_reviews, null, 2)}`) - } return pub.org_reviews[0].review_type } else { return 'pending' From d4261e18986e8770bcc556838a903d7cf8f4f174 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 12 Aug 2021 10:58:30 -0400 Subject: [PATCH 35/43] fix clear author selected if change center review --- client/src/pages/CenterReview.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index 2f0b08f0..754db40a 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -531,6 +531,7 @@ export default { }, selectedCenter: function () { console.log('Selected Center Change. Reloading publications...') + this.selectedCenterAuthor = this.preferredSelectedCenterAuthor this.loadPublications() }, changedPubYears: async function () { @@ -620,7 +621,7 @@ export default { const notInPersonPub2SetId = this.notInPersonPubSet(personPub2Id) const personPubSet1Id = this.getPersonPubSetId(personPub1Id) const personPubSet2Id = this.getPersonPubSetId(personPub2Id) - console.log(`Linking person pub id 1: ${personPub1Id} to person pub id 2: ${personPub2Id}`) + // console.log(`Linking person pub id 1: ${personPub1Id} to person pub id 2: ${personPub2Id}`) if (notInPersonPub1SetId && notInPersonPub2SetId) { // neither one is in a set yet and just add to set list // console.log(`Starting new set for personPubId1: ${personPub1Id}`) @@ -1259,7 +1260,6 @@ export default { this.publications = [] this.citationsByTitle = {} this.people = [] - this.selectedCenterAuthor = 'All' await this.loadCenterAuthorOptions() this.personPublicationsCombinedMatches = [] this.personPublicationsCombinedMatchesByReview = {} From 9c30c821d26440f05e86dfc2ae857a48334c0821 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Thu, 12 Aug 2021 21:00:38 -0400 Subject: [PATCH 36/43] update_synchronize_reviews use publication graph --- ingest/modules/confidenceSet.ts | 9 + ingest/modules/normedPersonPublication.ts | 22 ++ ingest/modules/publicationGraph.ts | 278 ++++++++++++++++++++++ ingest/modules/publicationSet.ts | 9 + ingest/synchronizeReviewStates.ts | 100 ++++---- 5 files changed, 365 insertions(+), 53 deletions(-) create mode 100644 ingest/modules/confidenceSet.ts create mode 100644 ingest/modules/normedPersonPublication.ts create mode 100644 ingest/modules/publicationGraph.ts create mode 100644 ingest/modules/publicationSet.ts diff --git a/ingest/modules/confidenceSet.ts b/ingest/modules/confidenceSet.ts new file mode 100644 index 00000000..2b6cf1c9 --- /dev/null +++ b/ingest/modules/confidenceSet.ts @@ -0,0 +1,9 @@ +import _ from 'lodash' +import NormedPerson from './normedPerson' +import NormedPublication from './normedPublication' + +export default interface ConfidenceSet { + // ------ begin declare properties used when using NormedPerson like an interface + value: Number + // ------ end declare properties used when using NormedPerson like an interface +} \ No newline at end of file diff --git a/ingest/modules/normedPersonPublication.ts b/ingest/modules/normedPersonPublication.ts new file mode 100644 index 00000000..fa31ac72 --- /dev/null +++ b/ingest/modules/normedPersonPublication.ts @@ -0,0 +1,22 @@ +import _ from 'lodash' +import ConfidenceSet from './confidenceSet' + +export default class NormedPersonPublication { + // ------ begin declare properties used when using NormedPerson like an interface + id: Number + person + publication + confidence?: Number + reviewTypeStatus?: string + mostRecentReview? + confidenceSet?: ConfidenceSet + // ------ end declare properties used when using NormedPerson like an interface + + public static getPublicationConfidence (normedPersonPub: NormedPersonPublication): Number { + if (normedPersonPub.confidenceSet) { + return normedPersonPub.confidenceSet.value + } else { + return normedPersonPub.confidence + } + } +} \ No newline at end of file diff --git a/ingest/modules/publicationGraph.ts b/ingest/modules/publicationGraph.ts new file mode 100644 index 00000000..96d30480 --- /dev/null +++ b/ingest/modules/publicationGraph.ts @@ -0,0 +1,278 @@ +import _ from 'lodash' +import PublicationSet from './publicationSet' +import NormedPersonPublication from './normedPersonPublication' + +export default class PublicationGraph { + + // these are helper objects to connect personPubSets together + // PersonPubId -> Person Pub Set ID Pointers + private personPubSetPointer: {} + // PersonPubSet Id -> Person Pub Id list (i.e., the set itself) + personPubSetsById: {} + // the current index for personPubSets, will increment whenever adding a new one + personPubSetIdIndex: Number + personPublicationsById: {} + + constructor () { + this.personPubSetIdIndex = 0 + this.personPubSetPointer = {} + this.personPubSetsById = {} + this.personPublicationsById = {} + } + + addToGraph (personPubs: NormedPersonPublication[]) { + // group by title + // group by doi + // then insert each set into graph and combine parent set nodes as needed + // seed the personPublication keys + const personPublicationsKeys = {} + _.each(personPubs, (personPub) => { + personPublicationsKeys[`${personPub.id}`] = { + titleKey: this.getPublicationTitleKey(personPub.publication.title), + doiKey: this.getPublicationDoiKey(personPub.publication) + } + }) + const publicationsGroupedByTitle = _.groupBy(personPubs, (personPub) => { + // let title = personPub.publication.title + const titleKey = personPublicationsKeys[`${personPub.id}`].titleKey + return `${titleKey}` + }) + + const publicationsGroupedByDoi = _.groupBy(personPubs, (personPub) => { + // let doi = personPub.publication.doi + const doiKey = personPublicationsKeys[`${personPub.id}`].doiKey + return `${doiKey}` + }) + + console.log(`Create publication graph...`) + // keep a map of personPubId to set id in order to find the set that something should be added to if found as same pub + // merge personPubs together by title and then doi + _.each(_.keys(publicationsGroupedByTitle), (titleKey) => { + this.linkPersonPubs(publicationsGroupedByTitle[titleKey]) + }) + + // now link together if same doi (if already found above will add to existing set) + _.each(_.keys(publicationsGroupedByDoi), (doiKey) => { + this.linkPersonPubs(publicationsGroupedByDoi[doiKey]) + }) + console.log(`Finished publication graph.`) + } + + getAllPublicationSets(): PublicationSet[] { + return _.values(this.personPubSetsById) + } + + getPublicationTitleKey (title) { + // normalize the string and remove characters like dashes as well + return this.normalizeString(title, true, true) + } + + getPublicationDoiKey (publication) { + if (!publication.doi) { + return `${publication.source_name}_${publication.source_id}` + } else { + return publication.doi + } + } + + removeSpaces (value) { + if (_.isString(value)) { + return _.clone(value).replace(/\s/g, '') + } else { + return value + } + } + // replace diacritics with alphabetic character equivalents + normalizeString (value, lowerCase, removeSpaces) { + if (_.isString(value)) { + let newValue = _.clone(value) + .normalize('NFD') + .replace(/[\u0300-\u036f]/g, '') // Remove diacritics + .replace(/[\u2019]/g, '\u0027') // the u0027 also normalizes the curly apostrophe to the straight one + .replace(/[&\\#,+()$~%.'":*?<>{}!-]/g, '') // remove periods and other remaining special characters + if (lowerCase) { + newValue = _.lowerCase(newValue) + } + if (removeSpaces) { + return this.removeSpaces(newValue) + } else { + return newValue + } + } else { + return value + } + } + + // will link all personPubs in this list together + linkPersonPubs (personPubList: NormedPersonPublication[]) { + // link all person pubs together in this list + const totalPubs = personPubList.length + _.each(personPubList, (personPub: NormedPersonPublication, index) => { + // at last item do nothing + try { + if (index === 0 && totalPubs === 1) { + // console.log(`Linking index: ${index}, only one pub in set, personpub: ${JSON.stringify(personPub['id'], null, 2)}`) + // will start a new personpub set list if not already in one + // console.log(`Passing person pub id to start a new set: ${personPub['id']}`) + this.startPersonPubSet(personPub) + } else if (index !== (totalPubs - 1)) { + // console.log(`Linking index: ${index} to ${index + 1} of ${totalPubs}), personpub: ${JSON.stringify(personPub['id'], null, 2)}`) + const nextPersonPub: NormedPersonPublication = _.nth(personPubList, (index + 1)) + this.linkPersonPubPair(personPub, nextPersonPub) + } + } catch (error) { + console.log(`Warning, error on linking publications: ${error}`) + throw error + } + }) + } + + getPersonPubSet (setId: Number): PublicationSet { + return this.personPubSetsById[`${setId}`] + } + + // this method will link person pubs by putting them in a person pub set + // together. If neither are already in a person pub set, they will be grouped together in a new set + // If only one has a set, the other will be added to that set + // If both are currently within a set, it will merge the two sets together + private linkPersonPubPair (personPub1: NormedPersonPublication, personPub2: NormedPersonPublication) { + const notInPersonPub1SetId: boolean = this.notInPersonPubSet(personPub1.id) + const notInPersonPub2SetId: boolean = this.notInPersonPubSet(personPub2.id) + const personPubSet1Id: Number = this.getPersonPubSetId(personPub1) + const personPubSet2Id: Number = this.getPersonPubSetId(personPub2) + // console.log(`Linking person pub id 1: ${personPub1.id} to person pub id 2: ${personPub2.id}`) + if (notInPersonPub1SetId && notInPersonPub2SetId) { + // neither one is in a set yet and just add to set list + // console.log(`Starting new set for personPubId1: ${personPub1.id}`) + const newSetId: Number = this.startPersonPubSet(personPub1) + // console.log(`Created personPubset1: ${newSetId} and added person pub1: ${personPub1.id}`) + // console.log(`Adding personPubId2: ${personPub2.id} to set1id: ${newSetId}`) + this.addPersonPubToSet(newSetId, personPub2) + } else if (notInPersonPub2SetId) { + // console.log(`Adding personPubId2: ${personPub2.id} to set1id: ${personPubSet1Id}`) + this.addPersonPubToSet(personPubSet1Id, personPub2) + } else if (notInPersonPub1SetId) { + // console.log(`Adding personPubId1: ${personPub1.id} to set2id: ${personPubSet2Id}`) + this.addPersonPubToSet(personPubSet2Id, personPub1) + } else { + // they are both in existing sets and need to merge them + // do nothing if they have the same pubsetid as they are already + // in the same set + // console.log(`Merging sets for personPubId1: ${personPub1.id} and personPubId2: ${personPub2.id}, set2Id->set1Id: (${personPubSet2Id}->${personPubSet1Id})`) + this.mergePersonPubSets(personPubSet1Id, personPubSet2Id) + } + } + + private mergePersonPubSets (set1Id: Number, set2Id: Number) { + // do nothing if they are the same set id + // console.log(`Merging person pub sets set1: ${set1Id} set2: ${set2Id}`) + if (set1Id !== set2Id) { + // add items from set2 into set1 if not already there, assumes everything is up to date with pointers + const set1: PublicationSet = this.getPersonPubSet(set1Id) + const set2: PublicationSet = this.getPersonPubSet(set2Id) + if (set1.reviewType !== set2.reviewType) { + const error = `Warning: Mismatch in reviewType for sets to be merged. found set 1: ${set1.reviewType} set 2: ${set2.reviewType}` + console.log(error) + } + const set2List: NormedPersonPublication[] = set2.personPublications + _.each(set2List, (personPub: NormedPersonPublication) => { + this.addPersonPubToSet(set1Id, personPub) + }) + // destroy the set2List + this.removePersonPubSet(set2Id) + } + } + + private removePersonPubSet (setId: Number) { + // only works if personPubs in this set already pointing to another one, else throw error + // do nothing if set already gone + const set = this.getPersonPubSet(setId) + if (set) { + _.each(set.personPublications, (personPub) => { + if (setId && this.getPersonPubSetId(personPub) === setId) { + const error = `Cannot remove person Pub Set (on merge), personPubId: ${personPub.id} not in any other set` + console.log(error) + throw error + } + }) + // if we get this far no errors encountered, and all person pubs are now in another set + // go ahead and delete it + _.unset(this.personPubSetsById, `${setId}`) + } + } + + private addPersonPubToSet (setId: Number, personPub: NormedPersonPublication) { + const setIdKey = `${setId}` + const personPubIdKey = `${personPub.id}` + // proceed if set exists + const set = this.getPersonPubSet(setId) + if (set) { + // do nothing if already in the set + if (this.getPersonPubSetId(personPub) !== setId) { + if (set.reviewType !== personPub.reviewTypeStatus) { + const error = `Warning person pub added to set with mismatched review types. Expected ${personPub.reviewTypeStatus}, found set type: ${set.reviewType}` + console.log(error) + } + this.personPubSetsById[setIdKey].personPublicationIds = _.concat(this.personPubSetsById[setIdKey].personPublicationIds, personPub.id) + this.personPubSetsById[setIdKey].personPublications = _.concat(this.personPubSetsById[setIdKey].personPublications, personPub) + this.personPubSetPointer[personPubIdKey] = setId + // console.log(`After add personpub: ${personPubId} to setid: ${setId} pubset pointers are: ${JSON.stringify(this.personPubSetPointer, null, 2)}`) + const mainPersonPub: NormedPersonPublication = set.mainPersonPub + if (!set.mainPersonPubId || NormedPersonPublication.getPublicationConfidence(mainPersonPub) < NormedPersonPublication.getPublicationConfidence(personPub)) { + _.set(set, 'mainPersonPub', personPub) + _.set(set, 'mainPersonPubId', personPub.id) + } + } + } else { + const error = `Failed to add personPub with id: ${personPub.id} to set id: ${setId}, personPubSet does not exist` + console.log(error) + throw error + } + } + + private notInPersonPubSet (personPubId: Number) { + // true if already in a set + const key = `${personPubId}` + return !this.personPubSetPointer[key] + } + + // this method is not currently thread-safe + // creates a new person pub set if one does not already exist for the given + // person Pub Id + private startPersonPubSet (personPub: NormedPersonPublication): Number { + if (this.notInPersonPubSet(personPub.id)) { + // console.log(`Creating person pub set for pub id: ${personPub.id}`) + const personPubSetId: Number = this.getNextPersonPubSetId() + this.personPubSetPointer[`${personPub.id}`] = personPubSetId + const pubSet: PublicationSet = { + personPublicationIds: [personPub.id], + personPublications: [personPub], + mainPersonPubId: personPub.id, + mainPersonPub: personPub, + reviewType: personPub.reviewTypeStatus + } + this.personPubSetsById[`${personPubSetId}`] = pubSet + return personPubSetId + } else { + const currentSetId: Number = this.getPersonPubSetId(personPub) + const currentSet: PublicationSet = this.getPersonPubSet(currentSetId) + if (currentSet.reviewType !== personPub.reviewTypeStatus) { + const error = `Warning: Mismatch on review type for person Pub set for personPub id: ${personPub.id}, expected review type: ${personPub.reviewTypeStatus} and found review type: ${currentSet.reviewType}` + console.log(error) + } else { + return this.getPersonPubSetId(personPub) + } + } + } + + // returns a person Pub set if it exists for that personPub, else returns undefined + private getPersonPubSetId (personPub: NormedPersonPublication): Number { + return this.personPubSetPointer[`${personPub.id}`] + } + + // this method is not currently thread-safe + private getNextPersonPubSetId (): Number { + this.personPubSetIdIndex = this.personPubSetIdIndex.valueOf() + 1 + return this.personPubSetIdIndex + } +} \ No newline at end of file diff --git a/ingest/modules/publicationSet.ts b/ingest/modules/publicationSet.ts new file mode 100644 index 00000000..fd1ddb36 --- /dev/null +++ b/ingest/modules/publicationSet.ts @@ -0,0 +1,9 @@ +import NormedPersonPublication from './normedPersonPublication' + +export default interface PublicationSet { + personPublicationIds: Number[], + personPublications: NormedPersonPublication[], + mainPersonPubId: Number, + mainPersonPub: NormedPersonPublication, + reviewType?: string +} \ No newline at end of file diff --git a/ingest/synchronizeReviewStates.ts b/ingest/synchronizeReviewStates.ts index dba90610..a075927a 100644 --- a/ingest/synchronizeReviewStates.ts +++ b/ingest/synchronizeReviewStates.ts @@ -12,6 +12,9 @@ import readReviewTypes from './gql/readReviewTypes' import insertReview from '../client/src/gql/insertReview' import readPersonPublicationsReviews from './gql/readPersonPublicationsReviews' import readOrganizations from './gql/readOrganizations' +import NormedPersonPublication from './modules/normedPersonPublication' +import PublicationGraph from './modules/publicationGraph' +import PublicationSet from './modules/publicationSet' dotenv.config({ path: '../.env' @@ -73,51 +76,46 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { return personPub }) - // console.log(`Start group by publications for person id: ${person.id}...`) - const publicationsGroupedByReview = _.groupBy(publications, function (pub) { - if (pub['reviews_aggregate'].nodes && pub['reviews_aggregate'].nodes.length > 0) { - return pub['reviews_aggregate'].nodes[0].review_type + const normedPersonPubs: NormedPersonPublication[] = _.map(publications, (personPub) => { + let reviewTypeStatus + let mostRecentReview + if (personPub['reviews_aggregate'].nodes && personPub['reviews_aggregate'].nodes.length > 0) { + reviewTypeStatus = personPub['reviews_aggregate'].nodes[0].review_type + mostRecentReview = personPub['reviews_aggregate'].nodes[0] } else { - return 'pending' + reviewTypeStatus = 'pending' } + const normedPersonPub: NormedPersonPublication = { + id: personPub.id, + person: personPub.person, + publication: personPub.publication, + reviewTypeStatus: reviewTypeStatus, + mostRecentReview: mostRecentReview + } + return normedPersonPub }) - // console.log(`Finish group by publications for person id: ${person.id}.`) - - // check for any title's with reviews out of sync, - // if more than one review type found add title mapped to array of reviewtype to array pub list - let publicationTitlesByReviewType = {} - - // put in pubs grouped by doi for each review status - _.each(reviewStates, (reviewType) => { - const publications = publicationsGroupedByReview[reviewType] - _.each(publications, (personPub) => { - if (personPub['publication'].title !== null){ - const titleKey = getPublicationTitleKey(personPub['publication'].title) - if (!publicationTitlesByReviewType[titleKey]) { - publicationTitlesByReviewType[titleKey] = {} - } - if (!publicationTitlesByReviewType[titleKey][reviewType]) { - publicationTitlesByReviewType[titleKey][reviewType] = [] - } - publicationTitlesByReviewType[titleKey][reviewType].push(personPub) - } - }) - }) + const pubGraph: PublicationGraph = new PublicationGraph() + pubGraph.addToGraph(normedPersonPubs) + + const pubSets: PublicationSet[] = pubGraph.getAllPublicationSets() // check for any title's with reviews out of sync - const publicationTitlesOutOfSync = [] + const publicationSetsOutOfSync: PublicationSet[] = [] - _.each(_.keys(publicationTitlesByReviewType), (titleKey) => { - if (_.keys(publicationTitlesByReviewType[titleKey]).length > 1) { - // console.log(`Warning: Doi out of sync found: ${doi} for person id: ${this.person.id} doi record: ${JSON.stringify(publicationDoisByReviewType[doi], null, 2)}`) - publicationTitlesOutOfSync.push(titleKey) - } + _.each(pubSets, (pubSet: PublicationSet) => { + let outOfSync = false + _.each (pubSet.personPublications, (personPub: NormedPersonPublication) => { + if (personPub.reviewTypeStatus !== pubSet.reviewType) { + outOfSync = true + } + }) + if (outOfSync) publicationSetsOutOfSync.push(pubSet) }) - if (publicationTitlesOutOfSync.length > 0) { - console.log(`Person id '${person['id']}' Dois found with reviews out of sync: ${JSON.stringify(publicationTitlesOutOfSync, null, 2)}`) + if (publicationSetsOutOfSync.length > 0) { + console.log(`Person id '${person['id']}' Dois found with reviews out of sync: ${JSON.stringify(publicationSetsOutOfSync, null, 2)}`) console.log(`Synchronizing Reviews for these personPubs out of sync for person id '${person['id']}'...`) - await pMap(publicationTitlesOutOfSync, async (titleKey) => { + await pMap(publicationSetsOutOfSync, async (pubSet: PublicationSet) => { // console.log(`Publication title '${title}' reviews by review type: ${JSON.stringify(publicationTitlesByReviewType[title], null, 2)}`) // get most recent review let mostRecentUpdateTime = undefined @@ -125,32 +123,28 @@ async function synchronizeReviewsForOrganization(persons, reviewOrgValue) { let mostRecentReview = undefined let newReviewStatus = undefined let newReviewOrgValue = undefined - _.each(_.keys(publicationTitlesByReviewType[titleKey]), (reviewType) => { - // just get first one - const personPub = publicationTitlesByReviewType[titleKey][reviewType][0] - if (personPub['reviews_aggregate'].nodes.length > 0) { + _.each(pubSet.personPublications, (personPub) => { + if (personPub.mostRecentReview) { // console.log(`Looking at reviews for doi: ${doi}, reviewType: ${reviewType}, nodes: ${JSON.stringify(personPub['reviews_aggregate'].nodes, null, 2)}`) - const currentDateTime = new Date(personPub['reviews_aggregate'].nodes[0].datetime) + const currentDateTime = new Date(personPub.mostRecentReview.datetime) if (!mostRecentUpdateTime || currentDateTime > mostRecentUpdateTime) { mostRecentUpdateTime = currentDateTime mostRecentReview = { - reviewType: reviewType, - reviewOrgValue: personPub['reviews_aggregate'].nodes[0].review_organization_value, - userId: personPub['reviews_aggregate'].nodes[0].user_id + reviewType: personPub.reviewTypeStatus, + reviewOrgValue: personPub.mostRecentReview.review_organization_value, + userId: personPub.mostRecentReview.user_id } } } }) if (mostRecentReview) { - await pMap (reviewStates, async (reviewType) => { - if (reviewType !== mostRecentReview.reviewType && publicationTitlesByReviewType[titleKey][reviewType]) { - await pMap (publicationTitlesByReviewType[titleKey][reviewType], async (personPub) => { - console.log(`Inserting Review for title key: ${titleKey}, personpub: '${personPub['id']}', most recent review ${JSON.stringify(mostRecentReview, null, 2)}`) - const mutateResult = await client.mutate( - insertReview(mostRecentReview.userId, personPub['id'], mostRecentReview.reviewType, mostRecentReview.reviewOrgValue) - ) - insertedReviewsCount += 1 - }, { concurrency: 1 }) + await pMap (pubSet.personPublications, async (personPub) => { + if (personPub.reviewTypeStatus !== mostRecentReview.reviewType) { + console.log(`Inserting Review personpub: '${personPub['id']}', most recent review ${JSON.stringify(mostRecentReview, null, 2)}`) + const mutateResult = await client.mutate( + insertReview(mostRecentReview.userId, personPub['id'], mostRecentReview.reviewType, mostRecentReview.reviewOrgValue) + ) + insertedReviewsCount += 1 } }, { concurrency: 1 }) } From 17de7d463c69f42c4d9e0732befc08c87780fcbc Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 13 Aug 2021 08:37:12 -0400 Subject: [PATCH 37/43] fix bug where pubsets were not getting cleared --- client/src/pages/CenterReview.vue | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index 754db40a..e2813a7d 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -448,7 +448,6 @@ export default { publicationsGroupedByTitleByOrgReview: {}, publicationsGroupedByDoiByInstitutionReview: {}, publicationsGroupedByDoiByOrgReview: {}, - publicationsGroupedByTitle: {}, sortAuthorsByTitle: {}, // map of title's to the matched author to sort by (i.e., the matched author with the lowest matched position) institutions: [], institutionGroup: [], @@ -1261,16 +1260,25 @@ export default { this.citationsByTitle = {} this.people = [] await this.loadCenterAuthorOptions() + this.personPubSetsById = {} + this.personPubSetPointer = {} + this.personPubSetIdIndex = 0 + this.personPublicationsById = {} + this.personPubSetsByReviewType = {} + this.personPublicationKeys = {} + this.publicationsGroupedByInstitutionReview = {} this.personPublicationsCombinedMatches = [] this.personPublicationsCombinedMatchesByReview = {} this.personPublicationsCombinedMatchesByOrgReview = {} this.filteredPersonPublicationsCombinedMatchesByOrgReview = {} this.publicationsGroupedByTitleByOrgReview = {} this.publicationsGroupedByTitleByInstitutionReview = {} - this.publicationsGroupedByTitle = {} + this.publicationsGroupedByDoiByOrgReview = {} + this.publicationsGroupedByDoiByInstitutionReview = {} this.confidenceSetItems = [] this.confidenceSet = undefined this.filteredPersonPubCounts = {} + this.sortAuthorsByTitle = {} }, async setCurrentPersonPublicationsCombinedMatches () { let reviewType = 'pending' From e3eb5f564525ed70559cf0b60d18bd084a5ef0cb Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 13 Aug 2021 13:18:03 -0400 Subject: [PATCH 38/43] fix bugreview sync with pubs with undefined doi --- client/src/pages/CenterReview.vue | 67 +++++++++++++++--------------- client/src/pages/Index.vue | 37 +++++++++++++---- ingest/modules/publicationGraph.ts | 40 ++++++++++++++---- 3 files changed, 95 insertions(+), 49 deletions(-) diff --git a/client/src/pages/CenterReview.vue b/client/src/pages/CenterReview.vue index e2813a7d..f082919f 100644 --- a/client/src/pages/CenterReview.vue +++ b/client/src/pages/CenterReview.vue @@ -607,7 +607,6 @@ export default { } } catch (error) { console.log(`Warning, error on linking publications: ${error}`) - throw error } }) }, @@ -650,9 +649,8 @@ export default { const set1 = this.getPersonPubSet(set1Id) const set2 = this.getPersonPubSet(set2Id) if (set1.reviewType !== reviewType || set2.reviewType !== reviewType) { - const error = `Error: Mismatch in reviewType for sets to be merged. Expected: ${reviewType}, found set 1: ${set1.reviewType} set 2: ${set2.reviewType}` + const error = `Warning: Mismatch in reviewType for sets to be merged. Expected: ${reviewType}, found set 1: ${set1.reviewType} set 2: ${set2.reviewType}` console.log(error) - throw error } const set2List = set2.personPublicationIds _.each(set2List, (personPubId) => { @@ -669,9 +667,8 @@ export default { if (set) { _.each(set, (personPubId) => { if (setId && this.getPersonPubSetId(personPubId) === setId) { - const error = `Cannot remove person Pub Set (on merge), personPubId: ${personPubId} not in any other set` + const error = `Warning: Cannot remove person Pub Set (on merge), personPubId: ${personPubId} not in any other set` console.log(error) - throw error } }) // if we get this far no errors encountered, and all person pubs are now in another set @@ -686,9 +683,8 @@ export default { // do nothing if already in the set if (this.getPersonPubSetId(personPubId) !== setId) { if (set.reviewType !== reviewType) { - const error = `Failed to add person pub to set with mismatched review types. Expected ${reviewType}, found: ${set.reviewType}` + const error = `Warning: Failed to add person pub to set with mismatched review types. Expected ${reviewType}, found: ${set.reviewType}` console.log(error) - throw error } const addPub = this.getPersonPublicationById(personPubId) this.personPubSetsById[setId].personPublicationIds = _.concat(this.personPubSetsById[setId].personPublicationIds, personPubId) @@ -702,9 +698,8 @@ export default { } } } else { - const error = `Failed to add personPub with id: ${personPubId} to set id: ${setId}, personPubSet does not exist` + const error = `Warning: Failed to add personPub with id: ${personPubId} to set id: ${setId}, personPubSet does not exist` console.log(error) - throw error } }, notInPersonPubSet (personPubId) { @@ -735,9 +730,8 @@ export default { const currentSetId = this.getPersonPubSetId(personPubId) const currentSet = this.getPersonPubSet(currentSetId) if (currentSet.reviewType !== reviewType) { - const error = `Error: Mismatch on review type for person Pub set for personPub id: ${personPubId}, expected review type: ${reviewType} and found review type: ${currentSet.reviewType}` + const error = `Warning: Mismatch on review type for person Pub set for personPub id: ${personPubId}, expected review type: ${reviewType} and found review type: ${currentSet.reviewType}` console.log(error) - throw error } else { return this.getPersonPubSetId(personPubId) } @@ -1303,11 +1297,15 @@ export default { return this.normalizeString(title, true, true) }, getPublicationDoiKey (publication) { - if (!publication.doi) { - return `${publication.source_name}_${publication.source_id}` + let doiKey + if (!publication.doi || publication.doi === null || this.removeSpaces(publication.doi) === '') { + if (publication.source_name && publication.source_id) { + doiKey = `${publication.source_name}_${publication.source_id}` + } } else { - return publication.doi + doiKey = publication.doi } + return doiKey }, getPubCSVResultObject (personPublication) { const titleKey = this.getPublicationTitleKey(personPublication.publication.title) @@ -1358,8 +1356,6 @@ export default { // start add code for pubsets // map both by shared title and by shared doi and merged lists together later - let publicationTitlesByReviewType = {} - let publicationDoisByReviewType = {} // put in pubs grouped by doi for each review status _.each(this.reviewStates, (reviewType) => { const publications = this.publicationsGroupedByInstitutionReview[reviewType] @@ -1374,29 +1370,18 @@ export default { this.publicationsGroupedByTitleByInstitutionReview[reviewType] = _.groupBy(publications, (personPub) => { // let title = personPub.publication.title const titleKey = this.personPublicationsKeys[personPub.id].titleKey - // console.log(`Title: '${title} title key: ${titleKey}'`) - if (!publicationTitlesByReviewType[titleKey]) { - publicationTitlesByReviewType[titleKey] = {} - } - if (!publicationTitlesByReviewType[titleKey][reviewType]) { - publicationTitlesByReviewType[titleKey][reviewType] = reviewType + if (titleKey) { + return `${titleKey}` + } else { + return undefined } - // publicationTitlesByReviewType[titleKey][reviewType].push(personPub) - return `${titleKey}` }) this.publicationsGroupedByDoiByInstitutionReview[reviewType] = _.groupBy(publications, (personPub) => { // let doi = personPub.publication.doi const doiKey = this.personPublicationsKeys[personPub.id].doiKey // console.log(`Doi: '${doi} doi key: ${doiKey}'`) - if (!publicationDoisByReviewType[doiKey]) { - publicationDoisByReviewType[doiKey] = {} - } - if (!publicationDoisByReviewType[doiKey][reviewType]) { - publicationDoisByReviewType[doiKey][reviewType] = reviewType - } - // publicationDoisByReviewType[doiKey][reviewType].push(personPub) - return `${doiKey}` + return doiKey }) // console.log(`Person pubs grouped by Title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) @@ -1407,12 +1392,26 @@ export default { // keep a map of personPubId to set id in order to find the set that something should be added to if found as same pub // merge personPubs together by title and then doi _.each(_.keys(this.publicationsGroupedByTitleByInstitutionReview[reviewType]), (titleKey) => { - this.linkPersonPubs(this.publicationsGroupedByTitleByInstitutionReview[reviewType][titleKey], reviewType) + if (titleKey && titleKey.length > 0) { + this.linkPersonPubs(this.publicationsGroupedByTitleByInstitutionReview[reviewType][titleKey], reviewType) + } else { + // make independent pub sets for each + _.each(this.publicationsGroupedByTitleByInstitutionReview[reviewType][titleKey], (pubSet) => { + this.linkPersonPubs([pubSet], reviewType) + }) + } }) // now link together if same doi (if already found above will add to existing set) _.each(_.keys(this.publicationsGroupedByDoiByInstitutionReview[reviewType]), (doiKey) => { - this.linkPersonPubs(this.publicationsGroupedByDoiByInstitutionReview[reviewType][doiKey], reviewType) + if (doiKey !== undefined && doiKey !== 'undefined' && doiKey !== null && this.removeSpaces(doiKey) !== '') { + this.linkPersonPubs(this.publicationsGroupedByDoiByInstitutionReview[reviewType][doiKey], reviewType) + } else { + // do separate pubset for each doi + _.each(this.publicationsGroupedByDoiByInstitutionReview[reviewType][doiKey], (pubSet) => { + this.linkPersonPubs([pubSet], reviewType) + }) + } }) console.log(`Finished publication graph.`) diff --git a/client/src/pages/Index.vue b/client/src/pages/Index.vue index 828f8343..f32f3f09 100644 --- a/client/src/pages/Index.vue +++ b/client/src/pages/Index.vue @@ -749,11 +749,15 @@ export default { return this.personPubSetIdIndex }, getPublicationDoiKey (publication) { - if (!publication.doi) { - return `${publication.source_name}_${publication.source_id}` + let doiKey + if (!publication.doi || publication.doi === null || this.removeSpaces(publication.doi) === '') { + if (publication.source_name && publication.source_id) { + doiKey = `${publication.source_name}_${publication.source_id}` + } } else { - return publication.doi + doiKey = publication.doi } + return doiKey }, async startProgressBar () { this.publicationsLoaded = false @@ -1292,8 +1296,11 @@ export default { if (!publicationTitlesByReviewType[titleKey][reviewType]) { publicationTitlesByReviewType[titleKey][reviewType] = reviewType } - // publicationTitlesByReviewType[titleKey][reviewType].push(personPub) - return `${titleKey}` + if (titleKey) { + return `${titleKey}` + } else { + return undefined + } }) this.publicationsGroupedByDoiByReview[reviewType] = _.groupBy(publications, (personPub) => { @@ -1307,7 +1314,7 @@ export default { publicationDoisByReviewType[doiKey][reviewType] = reviewType } // publicationDoisByReviewType[doiKey][reviewType].push(personPub) - return `${doiKey}` + return doiKey }) // console.log(`Person pubs grouped by Title are: ${JSON.stringify(this.publicationsGroupedByTitleByReview, null, 2)}`) @@ -1318,12 +1325,26 @@ export default { // keep a map of personPubId to set id in order to find the set that something should be added to if found as same pub // merge personPubs together by title and then doi _.each(_.keys(this.publicationsGroupedByTitleByReview[reviewType]), (titleKey) => { - this.linkPersonPubs(this.publicationsGroupedByTitleByReview[reviewType][titleKey], reviewType) + if (titleKey && titleKey.length > 0) { + this.linkPersonPubs(this.publicationsGroupedByTitleByReview[reviewType][titleKey], reviewType) + } else { + // make independent pub sets for each + _.each(this.publicationsGroupedByTitleByReview[reviewType][titleKey], (pubSet) => { + this.linkPersonPubs([pubSet], reviewType) + }) + } }) // now link together if same doi (if already found above will add to existing set) _.each(_.keys(this.publicationsGroupedByDoiByReview[reviewType]), (doiKey) => { - this.linkPersonPubs(this.publicationsGroupedByDoiByReview[reviewType][doiKey], reviewType) + if (doiKey !== undefined && doiKey !== 'undefined' && doiKey !== null && this.removeSpaces(doiKey) !== '') { + this.linkPersonPubs(this.publicationsGroupedByDoiByReview[reviewType][doiKey], reviewType) + } else { + // do separate pubset for each doi + _.each(this.publicationsGroupedByDoiByReview[reviewType][doiKey], (pubSet) => { + this.linkPersonPubs([pubSet], reviewType) + }) + } }) // get match with highest confidence level and use that one diff --git a/ingest/modules/publicationGraph.ts b/ingest/modules/publicationGraph.ts index 96d30480..79891d59 100644 --- a/ingest/modules/publicationGraph.ts +++ b/ingest/modules/publicationGraph.ts @@ -35,25 +35,45 @@ export default class PublicationGraph { const publicationsGroupedByTitle = _.groupBy(personPubs, (personPub) => { // let title = personPub.publication.title const titleKey = personPublicationsKeys[`${personPub.id}`].titleKey - return `${titleKey}` + if (titleKey) { + return `${titleKey}` + } else { + return undefined + } }) const publicationsGroupedByDoi = _.groupBy(personPubs, (personPub) => { // let doi = personPub.publication.doi const doiKey = personPublicationsKeys[`${personPub.id}`].doiKey - return `${doiKey}` + return doiKey }) console.log(`Create publication graph...`) // keep a map of personPubId to set id in order to find the set that something should be added to if found as same pub // merge personPubs together by title and then doi _.each(_.keys(publicationsGroupedByTitle), (titleKey) => { - this.linkPersonPubs(publicationsGroupedByTitle[titleKey]) + if (titleKey && titleKey.length > 0) { + console.log(`Linking person pubs by titleKey: '${titleKey}' personpubs: ${JSON.stringify(publicationsGroupedByTitle[titleKey], null, 2)}`) + this.linkPersonPubs(publicationsGroupedByTitle[titleKey]) + } else { + // make independent pub sets for each + _.each(publicationsGroupedByTitle[titleKey], (pubSet) => { + this.linkPersonPubs([pubSet]) + }) + } }) // now link together if same doi (if already found above will add to existing set) _.each(_.keys(publicationsGroupedByDoi), (doiKey) => { - this.linkPersonPubs(publicationsGroupedByDoi[doiKey]) + if (doiKey !== undefined && doiKey !== 'undefined' && doiKey !== null && this.removeSpaces(doiKey) !== '') { + console.log(`Linking person pubs by doiKey: '${doiKey}' personpubs: ${JSON.stringify(publicationsGroupedByDoi[doiKey], null, 2)}`) + this.linkPersonPubs(publicationsGroupedByDoi[doiKey]) + } else { + // do separate pubset for each doi + _.each(publicationsGroupedByDoi[doiKey], (pubSet) => { + this.linkPersonPubs([pubSet]) + }) + } }) console.log(`Finished publication graph.`) } @@ -68,11 +88,17 @@ export default class PublicationGraph { } getPublicationDoiKey (publication) { - if (!publication.doi) { - return `${publication.source_name}_${publication.source_id}` + console.log(`Generating doi key for publication id: ${publication.id} doi: ${publication.doi}`) + let doiKey + if (!publication.doi || publication.doi === null || this.removeSpaces(publication.doi) === '') { + if (publication.source_name && publication.source_id) { + doiKey = `${publication.source_name}_${publication.source_id}` + } } else { - return publication.doi + doiKey = publication.doi } + console.log(`Generated doi key for publication id: ${publication.id} doi: ${publication.doi} doi key: ${doiKey}`) + return doiKey } removeSpaces (value) { From 8abe2c5a1300332b08566bbc3714b2bf74367435 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 13 Aug 2021 13:30:47 -0400 Subject: [PATCH 39/43] remove comment --- ingest/modules/publicationGraph.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest/modules/publicationGraph.ts b/ingest/modules/publicationGraph.ts index 79891d59..d939b006 100644 --- a/ingest/modules/publicationGraph.ts +++ b/ingest/modules/publicationGraph.ts @@ -53,7 +53,7 @@ export default class PublicationGraph { // merge personPubs together by title and then doi _.each(_.keys(publicationsGroupedByTitle), (titleKey) => { if (titleKey && titleKey.length > 0) { - console.log(`Linking person pubs by titleKey: '${titleKey}' personpubs: ${JSON.stringify(publicationsGroupedByTitle[titleKey], null, 2)}`) + // console.log(`Linking person pubs by titleKey: '${titleKey}' personpubs: ${JSON.stringify(publicationsGroupedByTitle[titleKey], null, 2)}`) this.linkPersonPubs(publicationsGroupedByTitle[titleKey]) } else { // make independent pub sets for each From 69e7cc43e08b87856a28ab01c5a251736e9a0985 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 13 Aug 2021 13:34:06 -0400 Subject: [PATCH 40/43] add source_id to query for center review pubs --- ingest/gql/readPersonPublicationsReviews.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ingest/gql/readPersonPublicationsReviews.js b/ingest/gql/readPersonPublicationsReviews.js index 31f15ab2..4920bc41 100644 --- a/ingest/gql/readPersonPublicationsReviews.js +++ b/ingest/gql/readPersonPublicationsReviews.js @@ -15,6 +15,7 @@ export default function readPersonPublicationsReviews (id, orgValue) { title doi source_name + source_id } reviews_aggregate(where: {review_organization_value: {_eq: $review_organization_value}}, limit: 1, order_by: {datetime: desc}) { nodes { From 827de3b9a2da1bd2661f21c37e45d126da5b2296 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Fri, 13 Aug 2021 13:34:53 -0400 Subject: [PATCH 41/43] updated readme for setup --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40e88e27..6ea96275 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,20 @@ This pilot project will prototype a new process that automates data collection f ## Starting from scratch + clone from git into target machine + cd pace-admin cp .env.template .env + curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash + Logout of terminal and log back in + nvm install 13.14.0 + nvm use 13.14.0 + + Follow steps here to install docker: https://docs.docker.com/compose/install/ + make install - make cleardb - make start_docker + sudo snap install docker + sudo make cleardb + sudo make start_docker make migrate make newdb From f1302d395ab6be02348c6639eaab146709932908 Mon Sep 17 00:00:00 2001 From: rickjohnson Date: Tue, 17 Aug 2021 15:51:45 -0400 Subject: [PATCH 42/43] added toggle center in dashboard --- .../src/components/SearchView.vue | 63 ++++++++++++-- dashboard-search/src/ingest.ts | 85 ++++++++++--------- .../down.yaml | 6 ++ .../up.yaml | 8 ++ hasura/migrations/metadata.yaml | 6 +- 5 files changed, 119 insertions(+), 49 deletions(-) create mode 100644 hasura/migrations/1629225573738_add_relationship__table_public_reviews/down.yaml create mode 100644 hasura/migrations/1629225573738_add_relationship__table_public_reviews/up.yaml diff --git a/dashboard-client/src/components/SearchView.vue b/dashboard-client/src/components/SearchView.vue index e16adf1e..c006ca9b 100644 --- a/dashboard-client/src/components/SearchView.vue +++ b/dashboard-client/src/components/SearchView.vue @@ -20,12 +20,25 @@ {{cleanChip(option)}}
+
+
+ + + +
+
+ ({ personLoadCount: 0, @@ -439,7 +441,7 @@ export default { personScrollIndex: 0, dom, date, - firstModel: 375, + firstModel: 400, secondModel: 500, people: [], publications: [], @@ -1027,6 +1029,10 @@ export default { await this.$refs['personScroll'].scrollTo(scrollIndex) // console.log(this.$refs) this.$refs[`person${currentPersonIndex}`].show() + // check publications and if not loaded reload publications too + if (this.publications && this.publications.length <= 0) { + this.loadPublications(this.person) + } } else { console.log(`Person id: ${this.person.id} no longer found. Clearing UI states...`) // clear everything out