Skip to content

Make push work for hierarchical entries and bare-markdown layouts #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
amirhormati wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Make push work for hierarchical entries and bare-markdown layouts #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions toolbox/mdcode/src/libts/layouts/documents.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import * as md from '../metadata';
import { CatalogLayout } from '../layout';

const OVERVIEW_ASPECT_KEY = 'dataplex-types.global.overview';
const DEFAULT_ENTRY_TYPE = 'dataplex-types.global.generic';


export class DocumentsLayout implements CatalogLayout {
Expand Down Expand Up @@ -38,9 +39,8 @@ export class DocumentsLayout implements CatalogLayout {
try {
const content = await fs.promises.readFile(localPath, 'utf8');
const { entry } = parseMarkdown(content);
if (entry && entry.name) {
this._index.set(entry.name, localPath);
}
const name = entry?.name ?? deriveEntryNameFromPath(localPath, this._catalogPath);
this._index.set(name, localPath);
}
catch (err) {
// Skip unreadable/invalid files during indexing
Expand All @@ -63,12 +63,17 @@ export class DocumentsLayout implements CatalogLayout {
throw new Error(`Entry not found: ${name}`);
}
const content = await fs.promises.readFile(entryPath, 'utf8');
const { entry, body } = parseMarkdown(content);
const { entry: parsed, body } = parseMarkdown(content);

if (!entry) {
throw new Error(`Missing YAML frontmatter in Markdown file: ${entryPath}`);
const entry: md.Entry = parsed ?? ({ type: DEFAULT_ENTRY_TYPE, resource: {} } as md.Entry);
if (!entry.name) {
entry.name = name;
}


// Ensure the entry's type aspect is present — Dataplex create requires it.
entry.aspects = entry.aspects ?? {};
entry.aspects[entry.type] = entry.aspects[entry.type] ?? {};

const bodyTrimmed = body.trim();
if (bodyTrimmed) {
if (!entry.aspects) {
Expand Down Expand Up @@ -117,6 +122,11 @@ export class DocumentsLayout implements CatalogLayout {
}
}

function deriveEntryNameFromPath(absolutePath: string, catalogPath: string): string {
const rel = path.relative(catalogPath, absolutePath);
return rel.replace(/\.md$/, '');
}

export function parseMarkdown(content: string): { entry: md.Entry|null; body: string } {
const lines = content.split(/\r?\n/);
if (lines[0] !== '---') {
Expand All @@ -132,7 +142,10 @@ export function parseMarkdown(content: string): { entry: md.Entry|null; body: st
const body = lines.slice(endIndex + 1).join('\n');

const entry = (metadata.catalogEntry ?? {}) as md.Entry;
entry.type = metadata.type;
const declaredType = metadata.type;
entry.type = (typeof declaredType === 'string' && declaredType.startsWith('dataplex-types.'))
? declaredType
: DEFAULT_ENTRY_TYPE;
entry.resource = entry.resource ?? {}
entry.resource.displayName = metadata.title;
entry.resource.description = metadata.description;
Expand Down
7 changes: 2 additions & 5 deletions toolbox/mdcode/src/libts/sync.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,15 +81,12 @@ export class CatalogSync {

const exist = await this._catalog.lookupEntry(project, location, entry.name);
if (exist.status != 200 || !exist.result) {
console.log(`entry ${name} does not exist, will try to create the entry.`);

const entryGroup = nameParts[5];
const entryId = nameParts[7];
const entryId = nameParts.slice(7).join('/');
const createEntryRes = await this._catalog.createEntry(project, location, entryGroup, entryId, entry);
if (createEntryRes.status != 200 || !createEntryRes.result) {
console.log(`Failed to push entry ${entry.name}: Failed to create new entry.`);
return { success: false, details: `Failed to create entry ${entry.name}: ${createEntryRes.message || createEntryRes.status}` };
}
console.log(`Successfully created and pushed entry ${entry.name}`);
continue;
}

Expand Down