Merge pull request #34 from curvenote/fix/dedupe-bibtex

fix/dedupe bibtex

src/cli/services/export.ts

@@ -26,6 +26,13 @@ function makeTemplateOptionsOption() {
   );
 }
 
+function makeConverterOption() {
+  return new Option(
+    '-c, --converter <name>',
+    'Specify which converter to use for SVG processing (inkscape, imagemagick)',
+  ).default('inkscape');
+}
+
 function makeWordExportCLI(program: Command) {
   const command = new Command('docx')
     .alias('word')
@@ -56,6 +63,7 @@ function makeTexExportCLI(program: Command) {
     .addOption(makeImageOption())
     .addOption(makeTemplateOption())
     .addOption(makeTemplateOptionsOption())
+    .addOption(makeConverterOption())
     .action(clirun(oxaLinkToTex, { program }));
   return command;
 }
@@ -67,6 +75,7 @@ function makePdfExportCLI(program: Command) {
     .argument('[output]', 'The document filename to export to', 'main.pdf')
     .addOption(makeTemplateOption())
     .addOption(makeTemplateOptionsOption())
+    .addOption(makeConverterOption())
     .action(clirun(oxaLinkToPdf, { program }));
   return command;
 }

src/export/tex/index.ts

@@ -4,6 +4,7 @@ import { Blocks, VersionId, KINDS, oxaLink, convertToBlockId } from '@curvenote/
 import { toTex } from '@curvenote/schema';
 import os from 'os';
 import path from 'path';
+import { Logger } from 'logging';
 import { Block, Version } from '../../models';
 import { ISession } from '../../session/types';
 import { getChildren } from '../../actions/getChildren';
@@ -25,7 +26,8 @@ import {
   loadTemplateOptions,
   throwIfTemplateButNoJtex,
 } from './template';
-import { extractFirstFrameOfGif, isImageMagickAvailable } from '../utils/imagemagick';
+import * as inkscape from '../utils/inkscape';
+import * as imagemagick from '../utils/imagemagick';
 
 export function createTempFolder() {
   return fs.mkdtempSync(path.join(os.tmpdir(), 'curvenote'));
@@ -56,6 +58,47 @@ function writeBlocksToFile(
   fs.writeFileSync(filename, `${file}\n`);
 }
 
+function filterFilenamesByExtension(filenames: Record<string, string>, ext: string) {
+  return Object.entries(filenames).filter(([, filename]) => {
+    return path.extname(filename).toLowerCase() === ext;
+  });
+}
+
+/**
+ * Process images into supported formats
+ *
+ * @param session
+ * @param imageFilenames - updated via side effect
+ * @param originals
+ * @param convertFn
+ * @param buildPath
+ */
+async function processImages(
+  session: ISession,
+  imageFilenames: Record<string, string>,
+  originals: [string, string][],
+  convertFn: (orig: string, log: Logger, buildPath: string) => Promise<string | null>,
+  buildPath: string,
+) {
+  session.log.debug(`Processing ${originals.length} GIFs`);
+  const processed = await Promise.all(
+    originals.map(async ([key, orig]) => {
+      session.log.debug(`processing ${orig}`);
+      const png = await imagemagick.extractFirstFrameOfGif(orig, session.log, buildPath);
+      return { key, orig, png };
+    }),
+  );
+  processed.forEach(({ key, orig, png }) => {
+    if (png === null) {
+      session.log.error(
+        `Could not extract image from ${orig}, references to ${key} will be invalid`,
+      );
+      return;
+    }
+    imageFilenames[key] = png;
+  }, []);
+}
+
 export async function articleToTex(
   session: ISession,
   versionId: VersionId,
@@ -90,34 +133,47 @@ export async function articleToTex(
     buildPath,
   );
 
+  // TODO Dry up gif and svg processing
   session.log.debug('Processing GIFS if present...');
-  const gifs = Object.entries(imageFilenames).filter(([, filename]) => {
-    const ext = path.extname(filename);
-    return ext.toLowerCase() === '.gif';
-  });
+  const gifs = filterFilenamesByExtension(imageFilenames, '.gif');
   if (gifs.length > 0) {
-    if (!isImageMagickAvailable()) {
+    if (!imagemagick.isImageMagickAvailable()) {
       session.log.warn(
         'GIF images are references, but Imagemagick.convert not available to convert them. This may result in invalid output and/or an invalid pdf file',
       );
     } else {
       session.log.debug(`Processing ${gifs.length} GIFs`);
-      const processed = await Promise.all(
-        gifs.map(async ([key, gif]) => {
-          session.log.debug(`processing ${gif}`);
-          const png = await extractFirstFrameOfGif(gif, session.log, buildPath);
-          return { key, gif, png };
-        }),
+      await processImages(
+        session,
+        imageFilenames,
+        gifs,
+        imagemagick.extractFirstFrameOfGif,
+        buildPath,
       );
-      processed.forEach(({ key, gif, png }) => {
-        if (png === null) {
-          session.log.error(
-            `Could not extract image from ${gif}, references to ${key} will be invalid`,
-          );
-          return;
-        }
-        imageFilenames[key] = png;
-      }, []);
+    }
+  }
+
+  session.log.debug('Processing SVGs if present');
+  const svgs = filterFilenamesByExtension(imageFilenames, '.svg');
+  if (svgs.length > 0) {
+    if (opts.converter === 'imagemagick') {
+      if (!imagemagick.isImageMagickAvailable()) {
+        session.log.warn(
+          'SVGs need t be converted to pdf images, but imagemagick is not available to convert them. This may result in invalid output and/or an invalid pdf file',
+        );
+      } else {
+        session.log.debug(`Processing ${svgs.length} SVGs with IMAGEMAGICK to PNG`);
+      }
+      await processImages(session, imageFilenames, svgs, imagemagick.convertSVGToPNG, buildPath);
+    } else {
+      if (!inkscape.isInkscapeAvailable()) {
+        session.log.warn(
+          'SVGs need tobe converted to pdf images, but inkscape is not available to convert them. This may result in invalid output and/or an invalid pdf file',
+        );
+      } else {
+        session.log.debug(`Processing ${svgs.length} SVGs with INKSCAPE to PDF`);
+      }
+      await processImages(session, imageFilenames, svgs, inkscape.convertSVGToPDF, buildPath);
     }
   }
 

src/export/tex/types.ts

@@ -5,4 +5,5 @@ export interface TexExportOptions {
   options?: string;
   useBuildFolder?: boolean;
   texIsIntermediate?: boolean;
+  converter?: 'inkscape' | 'imagemagick';
 }

src/export/utils/imagemagick.ts

@@ -27,3 +27,24 @@ export async function extractFirstFrameOfGif(
   }
   return png;
 }
+
+export async function convertSVGToPNG(
+  svg: string,
+  log: Logger,
+  buildPath: string,
+): Promise<string | null> {
+  const dirname = path.dirname(svg);
+  const basename = path.basename(svg, path.extname(svg));
+  const png = path.join(dirname, `${basename}.png`);
+  const convert = makeExecutable(
+    `convert ${path.join(buildPath, svg)} -density 600 ${path.join(buildPath, png)}`,
+    log,
+  );
+  try {
+    await convert();
+  } catch (err) {
+    log.error(`Could not convert from SVG to PNG ${err}`);
+    return null;
+  }
+  return png;
+}

src/export/utils/inkscape.ts

@@ -0,0 +1,32 @@
+import { sync as which } from 'which';
+import path from 'path';
+import { Logger } from 'logging';
+import { makeExecutable } from './exec';
+
+export function isInkscapeAvailable() {
+  return which('inkscape', { nothrow: true });
+}
+
+export async function convertSVGToPDF(
+  svg: string,
+  log: Logger,
+  buildPath: string,
+): Promise<string | null> {
+  const dirname = path.dirname(svg);
+  const basename = path.basename(svg, path.extname(svg));
+  const pdf = path.join(dirname, `${basename}.pdf`);
+  const convert = makeExecutable(
+    `inkscape ${path.join(buildPath, svg)} --export-area-drawing --export-pdf=${path.join(
+      buildPath,
+      pdf,
+    )}`,
+    log,
+  );
+  try {
+    await convert();
+  } catch (err) {
+    log.error(`Could not convert SVG to PDF ${err}`);
+    return null;
+  }
+  return pdf;
+}

src/export/utils/walkArticle.ts

@@ -288,6 +288,7 @@ export async function loadImagesToBuffers(images: ArticleState['images']) {
       const { src } = getImageSrc(version);
       if (!src) return;
       const response = await fetch(src);
+      // TODO convert SVGs to PNG` with imagemagick
       const buffer = await response.buffer();
       buffers[key] = buffer;
     }),

src/export/utils/writeBibtex.ts

@@ -8,7 +8,18 @@ export async function writeBibtex(
   filename = 'main.bib',
   opts = { alwaysWriteFile: true },
 ) {
-  const bibliography = Object.entries(references).map(([, { bibtex }]) => bibtex);
+  const seen: string[] = [];
+  const bibliography = Object.entries(references)
+    .map(([, { label, bibtex }]) => {
+      if (seen.indexOf(label) !== -1) {
+        session.log.debug(`Dropping duplicate reference ${label}`);
+        return null;
+      }
+      seen.push(label);
+      return bibtex;
+    })
+    .filter((item: string | null) => item != null);
+
   if (bibliography.length === 0 && !opts.alwaysWriteFile) {
     session.log.debug('No references to write for the project.');
     return;