Skip to content

Commit

Permalink
fix: ignore pattern with language mapping in config (#753)
Browse files Browse the repository at this point in the history
  • Loading branch information
@katerina20
katerina20 authored Mar 28, 2024
1 parent e08595e commit bd66a66
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 37 deletions.
5 changes: 4 additions & 1 deletion src/main/java/com/crowdin/cli/commands/actions/UploadSourcesAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,10 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
throw e;
}

List<String> sources = SourcesUtils.getFiles(pb.getBasePath(), file.getSource(), file.getIgnore(), placeholderUtil)
LanguageMapping localLanguageMapping = LanguageMapping.fromConfigFileLanguageMapping(file.getLanguagesMapping());
LanguageMapping serverLanguageMapping = project.getLanguageMapping();
LanguageMapping languageMapping = LanguageMapping.populate(localLanguageMapping, serverLanguageMapping);
List<String> sources = SourcesUtils.getFiles(pb.getBasePath(), file.getSource(), file.getIgnore(), placeholderUtil, languageMapping)
.map(File::getAbsolutePath)
.collect(Collectors.toList());
String commonPath =
Expand Down
13 changes: 9 additions & 4 deletions src/main/java/com/crowdin/cli/commands/functionality/SourcesUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.crowdin.cli.commands.functionality;

import com.crowdin.cli.client.LanguageMapping;
import com.crowdin.cli.properties.helper.FileHelper;
import com.crowdin.cli.utils.PlaceholderUtil;
import com.crowdin.cli.utils.Utils;
Expand All @@ -19,19 +20,23 @@
import static java.util.Arrays.asList;

public class SourcesUtils {
public static Stream<File> getFiles(String basePath, String sourcePattern, List<String> ignorePattern, PlaceholderUtil placeholderUtil) {
public static Stream<File> getFiles(String basePath, String sourcePattern, List<String> ignorePattern, PlaceholderUtil placeholderUtil, LanguageMapping languageMapping) {
if (basePath == null || sourcePattern == null || placeholderUtil == null) {
throw new NullPointerException("null args in SourceUtils.getFiles");
}
FileHelper fileHelper = new FileHelper(basePath);
String relativePath = StringUtils.removeStart(sourcePattern, basePath);
List<File> sources = fileHelper.getFiles(relativePath);
List<String> formattedIgnores = placeholderUtil.format(sources, ignorePattern, false);
List<String> formattedIgnores = placeholderUtil.format(sources, ignorePattern, languageMapping);
return fileHelper.filterOutIgnoredFiles(sources, formattedIgnores)
.stream()
.filter(File::isFile);
}

public static Stream<File> getFiles(String basePath, String sourcePattern, List<String> ignorePattern, PlaceholderUtil placeholderUtil) {
return SourcesUtils.getFiles(basePath, sourcePattern, ignorePattern, placeholderUtil, null);
}

public static List<String> filterProjectFiles(
List<String> filePaths, String sourcePattern, List<String> ignorePatterns, boolean preserveHierarchy, PlaceholderUtil placeholderUtil
) {
Expand All @@ -44,7 +49,7 @@ public static List<String> filterProjectFiles(
Predicate<String> ignorePredicate;
if (preserveHierarchy) {
sourcePredicate = Pattern.compile("^" + PlaceholderUtil.formatSourcePatternForRegex(sourcePattern) + "$").asPredicate();
ignorePredicate = placeholderUtil.formatForRegex(ignorePatterns, false).stream()
ignorePredicate = placeholderUtil.formatForRegex(ignorePatterns).stream()
.map(Pattern::compile)
.map(Pattern::asPredicate)
.map(Predicate::negate)
Expand All @@ -71,7 +76,7 @@ public static List<String> filterProjectFiles(
};
ignorePredicate = ignorePatterns.stream()
.map(ignorePattern -> {
List<String> ignorePatternPaths = placeholderUtil.formatForRegex(asList(ignorePattern.split(Pattern.quote(File.separator))), false);
List<String> ignorePatternPaths = placeholderUtil.formatForRegex(asList(ignorePattern.split(Pattern.quote(File.separator))));
Collections.reverse(ignorePatternPaths);
return ignorePatternPaths;
})
Expand Down
50 changes: 32 additions & 18 deletions src/main/java/com/crowdin/cli/utils/PlaceholderUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,31 +80,46 @@ public PlaceholderUtil(List<Language> supportedLanguages, List<com.crowdin.clien
this.basePath = basePath;
}

public List<String> format(List<File> sources, List<String> toFormat, boolean onProjectLangs) {
public List<String> format(List<File> sources, List<String> toFormat) {
if (sources == null || toFormat == null) {
return new ArrayList<>();
}
List<String> res = new ArrayList<>();
for (String str : toFormat) {
res.addAll(this.format(sources, str, onProjectLangs));
res.addAll(this.format(sources, str, null));
}
return res;
}

public Set<String> format(List<File> sources, String toFormat, boolean onProjectLangs) {
public List<String> format(List<File> sources, List<String> toFormat, LanguageMapping languageMapping) {
if (sources == null || toFormat == null) {
return new HashSet<>();
return new ArrayList<>();
}
List<String> res = new ArrayList<>();
for (String str : toFormat) {
res.addAll(this.format(sources, str, languageMapping));
}
return res;
}

List<Language> languages = (onProjectLangs ? projectLanguages : supportedLanguages);
public Set<String> format(List<File> sources, String toFormat, LanguageMapping languageMapping) {
if (sources == null || toFormat == null) {
return new HashSet<>();
}

return languages.stream()
.map(lang -> this.replaceLanguageDependentPlaceholders(toFormat, lang))
return supportedLanguages.stream()
.map(lang -> languageMapping == null
? this.replaceLanguageDependentPlaceholders(toFormat, lang)
: this.replaceLanguageDependentPlaceholders(toFormat, languageMapping, lang))
.flatMap(changedToFormat -> sources.stream()
.map(source -> this.replaceFileDependentPlaceholders(changedToFormat, source)))
.collect(Collectors.toSet());
}

public Set<String> format(List<File> sources, String toFormat) {
return this.format(sources, toFormat, null);
}

public String replaceLanguageDependentPlaceholders(String toFormat, Language lang) {
if (toFormat == null || lang == null) {
throw new NullPointerException("null args in replaceLanguageDependentPlaceholders()");
Expand Down Expand Up @@ -206,18 +221,17 @@ public String replaceFileDependentPlaceholders(String toFormat, File file) {
return StringUtils.removeStart(toFormat, Utils.PATH_SEPARATOR);
}

public List<String> formatForRegex(List<String> toFormat, boolean onProjectLangs) {
List<Language> langs = (onProjectLangs) ? this.projectLanguages : this.supportedLanguages;
String langIds = langs.stream().map(Language::getId).collect(Collectors.joining("|", "(", ")"));
String langNames = langs.stream().map(Language::getName).collect(Collectors.joining("|", "(", ")"));
String langLocales = langs.stream().map(Language::getLocale).collect(Collectors.joining("|", "(", ")"));
String langLocalesWithUnderscore = langs.stream().map(Language::getLocale).map(s -> s.replace("-", "_"))
public List<String> formatForRegex(List<String> toFormat) {
String langIds = supportedLanguages.stream().map(Language::getId).collect(Collectors.joining("|", "(", ")"));
String langNames = supportedLanguages.stream().map(Language::getName).collect(Collectors.joining("|", "(", ")"));
String langLocales = supportedLanguages.stream().map(Language::getLocale).collect(Collectors.joining("|", "(", ")"));
String langLocalesWithUnderscore = supportedLanguages.stream().map(Language::getLocale).map(s -> s.replace("-", "_"))
.collect(Collectors.joining("|", "(", ")"));
String langTwoLettersCodes = langs.stream().map(Language::getTwoLettersCode).collect(Collectors.joining("|", "(", ")"));
String langThreeLettersCodes = langs.stream().map(Language::getThreeLettersCode).collect(Collectors.joining("|", "(", ")"));
String langAndroidCodes = langs.stream().map(Language::getAndroidCode).collect(Collectors.joining("|", "(", ")"));
String langOsxLocales = langs.stream().map(Language::getOsxLocale).collect(Collectors.joining("|", "(", ")"));
String langOsxCodes = langs.stream().map(Language::getOsxCode).collect(Collectors.joining("|", "(", ")"));
String langTwoLettersCodes = supportedLanguages.stream().map(Language::getTwoLettersCode).collect(Collectors.joining("|", "(", ")"));
String langThreeLettersCodes = supportedLanguages.stream().map(Language::getThreeLettersCode).collect(Collectors.joining("|", "(", ")"));
String langAndroidCodes = supportedLanguages.stream().map(Language::getAndroidCode).collect(Collectors.joining("|", "(", ")"));
String langOsxLocales = supportedLanguages.stream().map(Language::getOsxLocale).collect(Collectors.joining("|", "(", ")"));
String langOsxCodes = supportedLanguages.stream().map(Language::getOsxCode).collect(Collectors.joining("|", "(", ")"));
return toFormat.stream()
.map(PlaceholderUtil::formatSourcePatternForRegex)
.map(s -> s
Expand Down
26 changes: 23 additions & 3 deletions src/test/java/com/crowdin/cli/commands/functionality/SourcesUtilsTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.crowdin.cli.commands.functionality;

import com.crowdin.cli.client.LanguageMapping;
import com.crowdin.cli.properties.helper.TempProject;
import com.crowdin.cli.utils.PlaceholderUtilBuilder;
import com.crowdin.cli.utils.Utils;
Expand All @@ -16,9 +17,7 @@
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -91,6 +90,27 @@ static Stream<Arguments> testGetFiles1() {
);
}

@Test
public void testGetFiles_LanguageMapping() {
tempProject.addFile("folder/first.xml");
tempProject.addFile("folder/second.eng.txt");
tempProject.addFile("folder/second.xml");
HashMap<String, Map<String, String>> mapping = new HashMap<String, Map<String, String>>() {{
put("two_letters_code", new HashMap<String, String>() {{
put("en", "eng");
}}
);
}};
LanguageMapping languageMapping = LanguageMapping.fromConfigFileLanguageMapping(mapping);
Stream<File> sources = SourcesUtils.getFiles(
tempProject.getBasePath(),
"/folder/*",
Collections.singletonList("/folder/**/*.%two_letters_code%.*"),
PlaceholderUtilBuilder.STANDART.build(tempProject.getBasePath()),
languageMapping);
assertEquals(2, sources.count());
}

@ParameterizedTest
@MethodSource
@DisabledOnOs(OS.WINDOWS)
Expand Down
15 changes: 4 additions & 11 deletions src/test/java/com/crowdin/cli/utils/PlaceholderUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void testMainFunctionality(
"/proj/path/"
);

Set<String> result = placeholderUtil.format(Arrays.asList(sources), toFormat, true);
Set<String> result = placeholderUtil.format(Arrays.asList(sources), toFormat);

assertEquals(new HashSet<>(Arrays.asList(expected)), result);
}
Expand All @@ -53,13 +53,6 @@ static Stream<Arguments> testMainFunctionality() {
Utils.normalizePath("resources/%two_letters_code%_%original_file_name%"),
new String[] {Utils.normalizePath("resources/ua_messages.xml"), Utils.normalizePath("resources/ru_messages.xml")}
),
arguments(// Must be only de_messages
new Language[] { LanguageBuilder.DEU.build(), LanguageBuilder.ENG.build() },
new Language[] { LanguageBuilder.DEU.build() },
new File[] {new File("resources/messages.xml")},
Utils.normalizePath("resources/%two_letters_code%_%original_file_name%"),
new String[] {Utils.normalizePath("resources/de_messages.xml")}
),
arguments(// How to treat double asterisks
new Language[] {LanguageBuilder.ENG.build()},
new Language[] {LanguageBuilder.ENG.build()},
Expand Down Expand Up @@ -92,7 +85,7 @@ public void testMainFunctionalityWithLists(
"/proj/path/"
);

List<String> result = placeholderUtil.format(Arrays.asList(sources), Arrays.asList(toFormat), true);
List<String> result = placeholderUtil.format(Arrays.asList(sources), Arrays.asList(toFormat));

assertEquals(new HashSet<>(Arrays.asList(expected)), new HashSet<>(result));
}
Expand Down Expand Up @@ -122,8 +115,8 @@ public void testForNpe() {
assertThrows(NullPointerException.class, () -> new PlaceholderUtil(null, null, null));
PlaceholderUtil placeholderUtil = new PlaceholderUtil(new ArrayList<>(), new ArrayList<>(), "/here/it/goes/");

assertEquals(new ArrayList<String>(), placeholderUtil.format(null, new ArrayList<>(), false));
assertEquals(new HashSet<>(), placeholderUtil.format(null, "", false));
assertEquals(new ArrayList<String>(), placeholderUtil.format(null, new ArrayList<>()));
assertEquals(new HashSet<>(), placeholderUtil.format(null, ""));

assertThrows(NullPointerException.class, () -> placeholderUtil.replaceLanguageDependentPlaceholders(null, new Language()));
assertThrows(NullPointerException.class, () -> placeholderUtil.replaceLanguageDependentPlaceholders(null, null, null));
Expand Down

0 comments on commit bd66a66

Please sign in to comment.