Add '--all' option to download translations without local sources

build.gradle

@@ -53,7 +53,7 @@ dependencies {
     testCompileOnly 'org.projectlombok:lombok:1.18.12'
     testAnnotationProcessor 'org.projectlombok:lombok:1.18.12'
 
-    implementation 'info.picocli:picocli:4.5.0'
+    implementation 'info.picocli:picocli:4.6.1'
 
     implementation 'com.github.crowdin:crowdin-api-client-java:1.3.2'
 

src/main/java/com/crowdin/cli/client/CrowdinClientCore.java

@@ -104,6 +104,13 @@ protected static <T, R extends Exception> T executeRequest(Map<BiPredicate<Strin
         try {
             return r.get();
         } catch (HttpBadRequestException e) {
+            for (HttpBadRequestException.ErrorHolder eh : e.getErrors()) {
+                for (HttpBadRequestException.Error error : eh.getError().errors) {
+                    String code = (error.code != null) ? error.code : "<empty_code>";
+                    String message = (error.message != null) ? error.message : "<empty_message>";
+                    searchErrorHandler(errorHandlers, error.getCode(), error.getMessage());
+                }
+            }
             String errorMessage = "Wrong parameters: \n" + e.getErrors()
                 .stream()
                 .flatMap(holder -> holder.getError().getErrors()
@@ -114,22 +121,26 @@ protected static <T, R extends Exception> T executeRequest(Map<BiPredicate<Strin
         } catch (HttpException e) {
             String code = (e.getError().code != null) ? e.getError().code : "<empty_code>";
             String message = (e.getError().message != null) ? e.getError().message : "<empty_message>";
-            for (Map.Entry<BiPredicate<String, String>, R> errorHandler : errorHandlers.entrySet()) {
-                if (errorHandler.getKey().test(code, message)) {
-                    throw errorHandler.getValue();
-                }
-            }
-            for (Map.Entry<BiPredicate<String, String>, RuntimeException> errorHandler : standardErrorHandlers.entrySet()) {
-                if (errorHandler.getKey().test(code, message)) {
-                    throw errorHandler.getValue();
-                }
-            }
+            searchErrorHandler(errorHandlers, code, message);
             throw new RuntimeException(String.format("Error from server: <Code: %s, Message: %s>", code, message));
         } catch (Exception e) {
             throw e;
         }
     }
 
+    private static <T, R extends Exception> void searchErrorHandler(Map<BiPredicate<String, String>, R> errorHandlers, String code, String message) throws R {
+        for (Map.Entry<BiPredicate<String, String>, R> errorHandler : errorHandlers.entrySet()) {
+            if (errorHandler.getKey().test(code, message)) {
+                throw errorHandler.getValue();
+            }
+        }
+        for (Map.Entry<BiPredicate<String, String>, RuntimeException> errorHandler : standardErrorHandlers.entrySet()) {
+            if (errorHandler.getKey().test(code, message)) {
+                throw errorHandler.getValue();
+            }
+        }
+    }
+
     private static <T> List<T> unwrap(ResponseList<T> list) {
         return list
             .getData()

src/main/java/com/crowdin/cli/client/CrowdinProjectClient.java

@@ -159,9 +159,15 @@ public void editSource(Long fileId, List<PatchRequest> request) {
     }
 
     @Override
-    public void uploadTranslations(String languageId, UploadTranslationsRequest request) {
+    public void uploadTranslations(String languageId, UploadTranslationsRequest request) throws ResponseException {
+        Map<BiPredicate<String, String>, ResponseException> errorhandlers = new LinkedHashMap<BiPredicate<String, String>, ResponseException>() {{
+            put((code, message) -> code.equals("0") && message.equals("File is not allowed for language"),
+                new WrongLanguageException());
+            put((code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"),
+                new RepeatException());
+        }};
         executeRequestWithPossibleRetry(
-            (code, message) -> message.contains("File from storage with id #" + request.getStorageId() + " was not found"),
+            errorhandlers,
             () -> this.client.getTranslationsApi()
                 .uploadTranslations(this.projectId, languageId, request));
     }

src/main/java/com/crowdin/cli/client/CrowdinProjectInfo.java

@@ -46,6 +46,10 @@ void setLanguageMapping(LanguageMapping languageMapping) {
         this.languageMapping = languageMapping;
     }
 
+    /**
+     * Should be checked with isManagerAccess
+     * @return language mapping
+     */
     public LanguageMapping getLanguageMapping() {
         return this.languageMapping;
     }

src/main/java/com/crowdin/cli/client/ProjectClient.java

@@ -41,7 +41,7 @@ public interface ProjectClient extends Client {
 
     void editSource(Long fileId, List<PatchRequest> request);
 
-    void uploadTranslations(String languageId, UploadTranslationsRequest request);
+    void uploadTranslations(String languageId, UploadTranslationsRequest request) throws ResponseException;
 
     ProjectBuild startBuildingTranslation(BuildProjectTranslationRequest request);
 

src/main/java/com/crowdin/cli/client/WrongLanguageException.java

@@ -0,0 +1,5 @@
+package com.crowdin.cli.client;
+
+public class WrongLanguageException extends ResponseException {
+
+}

src/main/java/com/crowdin/cli/commands/Actions.java

@@ -21,7 +21,7 @@ public interface Actions {
 
     NewAction<PropertiesWithFiles, ProjectClient> download(
         FilesInterface files, boolean noProgress, String languageId, boolean pseudo, String branchName,
-        boolean ignoreMatch, boolean isVerbose, boolean plainView
+        boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean userServerSources
     );
 
     NewAction<NoProperties, NoClient> generate(FilesInterface files, Path destinationPath, boolean skipGenerateDescription);
@@ -35,7 +35,7 @@ NewAction<PropertiesWithFiles, ProjectClient> listSources(
         boolean noProgress, boolean treeView, boolean plainView);
 
     NewAction<PropertiesWithFiles, ProjectClient> listTranslations(
-        boolean noProgress, boolean treeView, boolean isLocal, boolean plainView);
+        boolean noProgress, boolean treeView, boolean isLocal, boolean plainView, boolean useServerSources, boolean withInContextLang);
 
     NewAction<PropertiesWithFiles, ProjectClient> status(
         boolean noProgress, String languageId, boolean isVerbose, boolean showTranslated, boolean showApproved);
@@ -78,7 +78,7 @@ NewAction<BaseProperties, ClientTm> tmDownload(
 
     NewAction<PropertiesWithTargets, ProjectClient> downloadTargets(
         List<String> targetNames, FilesInterface files, boolean noProgress,
-        List<String> langIds, boolean isVerbose, boolean plainView, boolean debug);
+        List<String> langIds, boolean isVerbose, boolean plainView, boolean debug, String branchName);
 
     NewAction<NoProperties, NoClient> checkNewVersion();
 

src/main/java/com/crowdin/cli/commands/actions/CliActions.java

@@ -24,9 +24,9 @@ public class CliActions implements Actions {
     @Override
     public NewAction<PropertiesWithFiles, ProjectClient> download(
         FilesInterface files, boolean noProgress, String languageId, boolean pseudo, String branchName,
-        boolean ignoreMatch, boolean isVerbose, boolean plainView
+        boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean useServerSources
     ) {
-        return new DownloadAction(files, noProgress, languageId, pseudo, branchName, ignoreMatch, isVerbose, plainView);
+        return new DownloadAction(files, noProgress, languageId, pseudo, branchName, ignoreMatch, isVerbose, plainView, useServerSources);
     }
 
     @Override
@@ -55,9 +55,9 @@ public NewAction<PropertiesWithFiles, ProjectClient> listSources(
 
     @Override
     public NewAction<PropertiesWithFiles, ProjectClient> listTranslations(
-        boolean noProgress, boolean treeView, boolean isLocal, boolean plainView
+        boolean noProgress, boolean treeView, boolean isLocal, boolean plainView, boolean useServerSources, boolean withInContextLang
     ) {
-        return new ListTranslationsAction(noProgress, treeView, isLocal, plainView);
+        return new ListTranslationsAction(noProgress, treeView, isLocal, plainView, useServerSources, withInContextLang);
     }
 
     @Override
@@ -157,9 +157,9 @@ public NewAction<NoProperties, NoClient> checkNewVersion() {
     @Override
     public NewAction<PropertiesWithTargets, ProjectClient> downloadTargets(
         List<String> targetNames, FilesInterface files, boolean noProgress,
-        List<String> langIds, boolean isVerbose, boolean plainView, boolean debug
+        List<String> langIds, boolean isVerbose, boolean plainView, boolean debug, String branchName
     ) {
-        return new DownloadTargetsAction(targetNames, files, noProgress, langIds, isVerbose, plainView, debug);
+        return new DownloadTargetsAction(targetNames, files, noProgress, langIds, isVerbose, plainView, debug, branchName);
     }
 
 

src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java

@@ -19,6 +19,7 @@
 import com.crowdin.cli.utils.console.ConsoleSpinner;
 import com.crowdin.client.languages.model.Language;
 import com.crowdin.client.sourcefiles.model.Branch;
+import com.crowdin.client.sourcefiles.model.FileInfo;
 import com.crowdin.client.translations.model.BuildProjectTranslationRequest;
 import com.crowdin.client.translations.model.CrowdinTranslationCreateProjectBuildForm;
 import com.crowdin.client.translations.model.ProjectBuild;
@@ -56,12 +57,13 @@ class DownloadAction implements NewAction<PropertiesWithFiles, ProjectClient> {
     private boolean ignoreMatch;
     private boolean isVerbose;
     private boolean plainView;
+    private boolean useServerSources;
 
     private Outputter out;
 
     public DownloadAction(
             FilesInterface files, boolean noProgress, String languageId, boolean pseudo, String branchName,
-            boolean ignoreMatch, boolean isVerbose, boolean plainView
+            boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean useServerSources
     ) {
         this.files = files;
         this.noProgress = noProgress || plainView;
@@ -71,6 +73,7 @@ public DownloadAction(
         this.ignoreMatch = ignoreMatch;
         this.isVerbose = isVerbose;
         this.plainView = plainView;
+        this.useServerSources = useServerSources;
     }
 
     @Override
@@ -91,6 +94,10 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
             }
         }
 
+        if (useServerSources && !pb.getPreserveHierarchy() && !plainView) {
+            out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("message.download_translations.preserve_hierarchy_warning")));
+        }
+
         PlaceholderUtil placeholderUtil =
             new PlaceholderUtil(
                 project.getSupportedLanguages(), project.getProjectLanguages(true), pb.getBasePath());
@@ -103,6 +110,8 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
             .map(br -> project.findBranchByName(br)
                 .orElseThrow(() -> new RuntimeException(RESOURCE_BUNDLE.getString("error.not_found_branch"))));
 
+        Map<String, com.crowdin.client.sourcefiles.model.File> serverSources = ProjectFilesUtils.buildFilePaths(project.getDirectories(), project.getFiles());
+
         LanguageMapping serverLanguageMapping = project.getLanguageMapping();
 
         Map<Pair<File, List<String>>, List<Map<String, String>>> fileBeansWithDownloadedFiles = new TreeMap<>();
@@ -120,7 +129,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
                     : RequestBuilder.crowdinTranslationCreateProjectPseudoBuildForm(true, null, null, null, null);
                 Pair<File, List<String>> downloadedFiles = this.download(request, client, pb.getBasePath());
                 for (FileBean fb : pb.getFiles()) {
-                    Map<String, String> filesWithMapping = this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil);
+                    Map<String, String> filesWithMapping = this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy());
                     fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles, new ArrayList<>());
                     fileBeansWithDownloadedFiles.get(downloadedFiles).add(filesWithMapping);
                 }
@@ -163,7 +172,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
                                 && fb.getExportApprovedOnly() == downloadConfiguration.getRight()) {
 
                                 Map<String, String> filesWithMapping =
-                                    this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil);
+                                    this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy());
                                 fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles, new ArrayList<>());
                                 fileBeansWithDownloadedFiles.get(downloadedFiles).add(filesWithMapping);
                             }
@@ -297,12 +306,20 @@ private Pair<File, List<String>> download(BuildProjectTranslationRequest request
     }
 
     private Map<String, String> getFiles(
-        FileBean fb, String basePath, LanguageMapping serverLanguageMapping, List<Language> forLanguages, PlaceholderUtil placeholderUtil
+        FileBean fb, String basePath, LanguageMapping serverLanguageMapping, List<Language> forLanguages, PlaceholderUtil placeholderUtil, List<String> allServerSources, boolean preserveHierarchy
     ) {
         List<String> sources =
             SourcesUtils.getFiles(basePath, fb.getSource(), fb.getIgnore(), placeholderUtil)
                 .map(File::getAbsolutePath)
                 .collect(Collectors.toList());
+        if (useServerSources) {
+            List<String> serverSources = SourcesUtils.filterProjectFiles(allServerSources, fb.getSource(), fb.getIgnore(), preserveHierarchy, placeholderUtil)
+                .stream()
+                .map(s -> Utils.joinPaths(basePath, s))
+                .filter(s -> !sources.contains(s))
+                .collect(Collectors.toList());
+            sources.addAll(serverSources);
+        }
         LanguageMapping localLanguageMapping = LanguageMapping.fromConfigFileLanguageMapping(fb.getLanguagesMapping());
         LanguageMapping languageMapping = LanguageMapping.populate(localLanguageMapping, serverLanguageMapping);
         Map<String, String> translationReplace =

src/main/java/com/crowdin/cli/commands/actions/DownloadTargetsAction.java

@@ -21,6 +21,7 @@
 import com.crowdin.client.translations.model.ExportProjectTranslationRequest;
 import lombok.NonNull;
 import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.io.IOException;
@@ -29,6 +30,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -46,10 +48,11 @@ class DownloadTargetsAction implements NewAction<PropertiesWithTargets, ProjectC
     private boolean isVerbose;
     private boolean plainView;
     private boolean debug;
+    private String branchName;
 
     public DownloadTargetsAction(
         @NonNull List<String> targetNames, FilesInterface files, boolean noProgress,
-        List<String> langIds, boolean isVerbose, boolean plainView, boolean debug
+        List<String> langIds, boolean isVerbose, boolean plainView, boolean debug, String branchName
     ) {
         this.targetNames = targetNames;
         this.files = files;
@@ -58,6 +61,7 @@ public DownloadTargetsAction(
         this.isVerbose = isVerbose;
         this.plainView = plainView;
         this.debug = debug;
+        this.branchName = branchName;
     }
 
     @Override
@@ -72,12 +76,26 @@ public void act(Outputter out, PropertiesWithTargets pb, ProjectClient client) {
             new PlaceholderUtil(
                 project.getSupportedLanguages(), project.getProjectLanguages(true), pb.getBasePath());
 
-        Map<String, Long> projectFiles = ProjectFilesUtils.buildFilePaths(project.getDirectories(), project.getBranches(), project.getFileInfos())
+        if (StringUtils.isNotEmpty(branchName) && !project.findBranchByName(branchName).isPresent()) {
+            throw new RuntimeException(RESOURCE_BUNDLE.getString("error.not_found_branch"));
+        }
+        Optional<Long> branchId = project.findBranchByName(branchName).map(Branch::getId);
+
+        Map<String, Long> projectFiles = (branchId.isPresent()
+            ? ProjectFilesUtils
+                .buildFilePaths(project.getDirectories(), ProjectFilesUtils.filterFilesByBranch(project.getFileInfos(), branchId.get()))
+            : ProjectFilesUtils.buildFilePaths(project.getDirectories(), project.getBranches(), project.getFileInfos()))
             .entrySet().stream()
             .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getId()));
-        Map<String, Long> projectDirs = ProjectFilesUtils.buildDirectoryPaths(project.getDirectories(), project.getBranches())
-            .entrySet().stream()
+
+        Map<String, Long> projectDirs = (branchId.isPresent()
+            ? ProjectFilesUtils.buildDirectoryPaths(project.getDirectories())
+                .entrySet().stream()
+                .filter(entry -> branchId.get().equals(project.getDirectories().get(entry.getKey()).getBranchId()))
+            : ProjectFilesUtils.buildDirectoryPaths(project.getDirectories(), project.getBranches())
+                .entrySet().stream())
             .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey));
+
         Map<String, Long> projectBranches = project.getBranches()
             .values().stream()
             .collect(Collectors.toMap(Branch::getName, Branch::getId));
@@ -188,7 +206,7 @@ public void act(Outputter out, PropertiesWithTargets pb, ProjectClient client) {
                     for (String langId : specifiedLangs) {
                         ExportProjectTranslationRequest request = RequestBuilder.exportProjectTranslation(templateRequest);
                         request.setTargetLanguageId(langId);
-                        String targetFileLang = placeholderUtil.replaceLanguageDependentPlaceholders(fb.getFile(), projectLanguages.get(langId));
+                        String targetFileLang = placeholderUtil.replaceLanguageDependentPlaceholders(fb.getFile(), project.getLanguageMapping(), projectLanguages.get(langId));
                         builtRequests.add(Pair.of(targetFileLang, request));
                     }
 

src/main/java/com/crowdin/cli/commands/actions/GlossaryListAction.java

@@ -31,12 +31,13 @@ public void act(Outputter out, BaseProperties pb, ClientGlossary client) {
             if (!plainView) {
                 out.println(OK.withIcon(
                     String.format(RESOURCE_BUNDLE.getString("message.glossary.list"), glossary.getName(), glossary.getId(), glossary.getTerms())));
-                if (isVerbose && (glossary.getTerms() == null || glossary.getTerms() > 0)) {
+                if (isVerbose && mayHaveTerms(glossary)) {
                     try {
                         List<Term> terms = client.listTerms(glossary.getId());
                         for (Term term : terms) {
+                            String description = (term.getDescription() != null) ? term.getDescription() : "";
                             out.println(String.format(
-                                RESOURCE_BUNDLE.getString("message.glossary.list_term"), term.getId(), term.getText(), term.getDescription()));
+                                RESOURCE_BUNDLE.getString("message.glossary.list_term"), term.getId(), term.getText(), description));
                         }
                     } catch (Exception e) {
                         out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("error.glossary.no_permission")));
@@ -47,4 +48,8 @@ public void act(Outputter out, BaseProperties pb, ClientGlossary client) {
             }
         }
     }
+
+    private boolean mayHaveTerms(Glossary glossary) {
+        return glossary != null && (glossary.getTerms() == null || glossary.getTerms() > 0);
+    }
 }