Skip to content

Commit

Permalink
1. reformat some code.
Browse files Browse the repository at this point in the history
2. fix some config.
  • Loading branch information
Alexkun committed Mar 5, 2022
1 parent 4c219ad commit 3a21bdc
Show file tree
Hide file tree
Showing 16 changed files with 194 additions and 111 deletions.
2 changes: 1 addition & 1 deletion assembly-combined-package/assembly-combined/bin/linkis-cli
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
export LANG=en_US.utf-8
#set -x

LINKIS_CLIENT='com.webank.wedatasphere.linkis.cli.application.LinkisClientApplication'
LINKIS_CLIENT='org.apache.linkis.cli.application.LinkisClientApplication'

i=0
for arg in "$@"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ wds.linkis.server.restful.scan.packages=org.apache.linkis.entrance.restful
wds.linkis.server.socket.mode=false
wds.linkis.entrance.config.log.path=hdfs:///appcom/logs/linkis
#wds.linkis.resultSet.store.path=hdfs:///tmp/linkis
wds.linkis.orchestrator.computation.operation.builder.class=com.webank.wedatasphere.linkis.orchestrator.operation.TuningOperationBuilder
wds.linkis.orchestrator.computation.operation.builder.class=org.apache.linkis.orchestrator.operation.TuningOperationBuilder


wds.linkis.entrance.shell.danger.check.enabled=false
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@
<root level="INFO">
<appender-ref ref="RollingFile"/>
</root>
<logger name="com.webank.wedatasphere.linkis.message.builder.DefaultMessageJob" level="warn" additivity="true">
<logger name="org.apache.linkis.message.builder.DefaultMessageJob" level="warn" additivity="true">
<appender-ref ref="RollingFile"/>
</logger>

<logger name="com.webank.wedatasphere.linkis.message.scheduler.DefaultMessageExecutor" level="warn" additivity="true">
<logger name="org.apache.linkis.message.scheduler.DefaultMessageExecutor" level="warn" additivity="true">
<appender-ref ref="RollingFile"/>
</logger>
<logger name="com.netflix.loadbalancer.DynamicServerListLoadBalancer" level="warn" additivity="true">
Expand Down
4 changes: 2 additions & 2 deletions assembly-combined-package/assembly-combined/conf/log4j2.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@
<root level="INFO">
<appender-ref ref="RollingFile"/>
</root>
<logger name="com.webank.wedatasphere.linkis.message.builder.DefaultMessageJob" level="warn" additivity="true">
<logger name="org.apache.linkis.message.builder.DefaultMessageJob" level="warn" additivity="true">
<appender-ref ref="RollingFile"/>
</logger>

<logger name="com.webank.wedatasphere.linkis.message.scheduler.DefaultMessageExecutor" level="warn" additivity="true">
<logger name="org.apache.linkis.message.scheduler.DefaultMessageExecutor" level="warn" additivity="true">
<appender-ref ref="RollingFile"/>
</logger>
<logger name="com.netflix.loadbalancer.DynamicServerListLoadBalancer" level="warn" additivity="true">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import java.lang.Boolean
import org.apache.linkis.common.conf.{CommonVars, Configuration, TimeType}
import org.apache.linkis.common.utils.{DESUtil, Logging, Utils}
import org.apache.linkis.server.exception.BDPInitServerException
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang3.StringUtils
import sun.misc.BASE64Encoder


Expand Down Expand Up @@ -99,8 +99,6 @@ object ServerConfiguration extends Logging{
val BDP_SERVER_WAR_TEMPDIR = CommonVars("wds.linkis.server.war.tempdir", new File(BDP_SERVER_HOME.getValue, "web/webapps").getPath)
val BDP_SERVER_SERVER_DEFAULT_DIR_ALLOWED = CommonVars("wds.linkis.server.default.dir.allowed", "false")
val BDP_SERVER_WEB_SESSION_TIMEOUT = CommonVars("wds.linkis.server.web.session.timeout", new TimeType("2h"))
//val LINKIS_SERVER_SESSION_TICKETID_KEY = CommonVars("wds.linkis.session.ticket.key", "linkis_user_session_ticket_id_v1")
val LINKIS_SERVER_SESSION_TICKETID_KEY = CommonVars("wds.linkis.session.ticket.key", "bdp-user-ticket-id")

val BDP_SERVER_EVENT_QUEUE_SIZE = CommonVars("wds.linkis.server.event.queue.size", 5000)
val BDP_SERVER_EVENT_CONSUMER_THREAD_SIZE = CommonVars("wds.linkis.server.event.consumer.thread", 10)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,64 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.linkis.engineconn.computation.executor.upstream.access;

import org.apache.linkis.DataWorkCloudApplication;
import org.apache.linkis.common.ServiceInstance;
import org.apache.linkis.common.conf.DWCArgumentsParser;
import org.apache.linkis.engineconn.common.creation.DefaultEngineCreationContext;
import org.apache.linkis.engineconn.common.creation.EngineCreationContext;
import org.apache.linkis.governance.common.conf.GovernanceCommonConf;
import org.apache.linkis.governance.common.utils.EngineConnArgumentsParser;
import org.apache.linkis.rpc.Sender;
import org.apache.linkis.server.conf.ServerConfiguration;
import org.apache.linkis.server.utils.LinkisMainHelper;

import org.apache.commons.lang3.StringUtils;

import org.junit.Before;
import org.junit.Test;
import org.springframework.cloud.client.discovery.DiscoveryClient;

import java.util.Arrays;
import java.util.List;

import static org.junit.Assert.*;

/**
* Created by shangda on 2022/2/9.
*/
public class ECTaskEntranceInfoAccessTest {

@Before
public void before() {
// System.getProperties().setProperty("wds.linkis.server.conf", "linkis-et-jobhistory-scan.properties");
// System.getProperties().setProperty("wds.linkis.server.conf",
// "linkis-et-jobhistory-scan.properties");
System.out.println("Spring is enabled, now try to start SpringBoot.");
System.out.println("<--------------------Start SpringBoot App-------------------->");
String existsExcludePackages = ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES().getValue();
if (!StringUtils.isEmpty(existsExcludePackages)) {
DataWorkCloudApplication.setProperty(ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES().key(), existsExcludePackages);
DataWorkCloudApplication.setProperty(
ServerConfiguration.BDP_SERVER_EXCLUDE_PACKAGES().key(), existsExcludePackages);
}

String[] args = new String[]{
"--spring-conf","eureka.client.serviceUrl.defaultZone=http://ip:port/eureka/",
"--spring-conf", "logging.config=classpath:log4j2.xml",
"--spring-conf", "spring.profiles.active=engineconn",
"--spring-conf", "server.port=28899",
"--spring-conf", "spring.application.name=linkis-cg-engineconn"};
String[] args =
new String[] {
"--spring-conf", "eureka.client.serviceUrl.defaultZone=http://ip:port/eureka/",
"--spring-conf", "logging.config=classpath:log4j2.xml",
"--spring-conf", "spring.profiles.active=engineconn",
"--spring-conf", "server.port=28899",
"--spring-conf", "spring.application.name=linkis-cg-engineconn"
};
// 加载spring类
try {
// ECTaskEntranceInfoAccessHelper.initApp(args);
// ECTaskEntranceInfoAccessHelper.initApp(args);
} catch (Exception e) {
System.out.println(e.getStackTrace());
}

ServiceInstance[] instances = Sender.getInstances(GovernanceCommonConf.ENGINE_CONN_SPRING_NAME().getValue());
ServiceInstance[] instances =
Sender.getInstances(GovernanceCommonConf.ENGINE_CONN_SPRING_NAME().getValue());

System.out.println("<--------------------SpringBoot App init succeed-------------------->");
}

@Test
public void main() throws Exception {


// LinkisJobHistoryScanApplication.main(new String[]{"2021122919", "2021122921"});
// LinkisJobHistoryScanApplication.main(new String[]{"2021122919", "2021122921"});
}

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,25 @@
@Component(value = CliHeartBeatMonitorAnnotation.BEAN_NAME)
public @interface CliHeartBeatMonitorAnnotation {
String BEAN_NAME = "cliHeartBeatMonitor";

@AliasFor(annotation = Component.class)
String value() default BEAN_NAME;

@Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
@Target({
ElementType.CONSTRUCTOR,
ElementType.FIELD,
ElementType.METHOD,
ElementType.TYPE,
ElementType.PARAMETER
})
@Retention(RetentionPolicy.RUNTIME)
@Qualifier(BEAN_NAME)
@Autowired
@interface CliHeartBeatMonitorAutowiredAnnotation {
@AliasFor(annotation = Qualifier.class)
String value() default BEAN_NAME;

@AliasFor(annotation = Autowired.class)
boolean required() default true;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,21 @@ public ResultSetEngine generateResultSetEngine() {

@CliHeartBeatMonitorAnnotation
@ConditionalOnMissingBean(name = {CliHeartBeatMonitorAnnotation.BEAN_NAME})
public CliHeartbeatMonitor generateCliHeartbeatMonitor(){
public CliHeartbeatMonitor generateCliHeartbeatMonitor() {
CliHeartbeatMonitor cliHeartbeatMonitor = new CliHeartbeatMonitor(new KillHandler());
cliHeartbeatMonitor.start();
return cliHeartbeatMonitor;
}

@PersistenceManagerBeanAnnotation
@ConditionalOnMissingBean(name = {PersistenceManagerBeanAnnotation.BEAN_NAME})
public PersistenceManager generatePersistenceManager(@PersistenceEngineBeanAnnotation.PersistenceEngineAutowiredAnnotation PersistenceEngine persistenceEngine,
@ResultSetEngineBeanAnnotation.ResultSetEngineAutowiredAnnotation ResultSetEngine resultSetEngine,
@CliHeartBeatMonitorAnnotation.CliHeartBeatMonitorAutowiredAnnotation CliHeartbeatMonitor cliHeartbeatMonitor){
public PersistenceManager generatePersistenceManager(
@PersistenceEngineBeanAnnotation.PersistenceEngineAutowiredAnnotation
PersistenceEngine persistenceEngine,
@ResultSetEngineBeanAnnotation.ResultSetEngineAutowiredAnnotation
ResultSetEngine resultSetEngine,
@CliHeartBeatMonitorAnnotation.CliHeartBeatMonitorAutowiredAnnotation
CliHeartbeatMonitor cliHeartbeatMonitor) {
logger.info("init PersistenceManager.");
QueryPersistenceManager persistenceManager = new QueryPersistenceManager();
persistenceManager.setPersistenceEngine(persistenceEngine);
Expand All @@ -99,14 +103,18 @@ public PersistenceManager generatePersistenceManager(@PersistenceEngineBeanAnnot

@EntranceParserBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceParserBeanAnnotation.BEAN_NAME})
public EntranceParser generateEntranceParser(@PersistenceManagerBeanAnnotation.PersistenceManagerAutowiredAnnotation PersistenceManager persistenceManager){
public EntranceParser generateEntranceParser(
@PersistenceManagerBeanAnnotation.PersistenceManagerAutowiredAnnotation
PersistenceManager persistenceManager) {
return new CommonEntranceParser(persistenceManager);
}

@EntranceListenerBusBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceListenerBusBeanAnnotation.BEAN_NAME})
public EntranceEventListenerBus<EntranceEventListener, EntranceEvent> generateEntranceEventListenerBus() {
EntranceEventListenerBus<EntranceEventListener, EntranceEvent> entranceEventListenerBus = new EntranceEventListenerBus<EntranceEventListener, EntranceEvent>();
public EntranceEventListenerBus<EntranceEventListener, EntranceEvent>
generateEntranceEventListenerBus() {
EntranceEventListenerBus<EntranceEventListener, EntranceEvent> entranceEventListenerBus =
new EntranceEventListenerBus<EntranceEventListener, EntranceEvent>();
entranceEventListenerBus.start();
return entranceEventListenerBus;
}
Expand Down Expand Up @@ -141,8 +149,11 @@ public EntranceInterceptor[] generateEntranceInterceptors() {

@ErrorCodeListenerBeanAnnotation
@ConditionalOnMissingBean(name = {ErrorCodeListenerBeanAnnotation.BEAN_NAME})
public ErrorCodeListener generateErrorCodeListener(@PersistenceManagerBeanAnnotation.PersistenceManagerAutowiredAnnotation PersistenceManager persistenceManager,
@EntranceParserBeanAnnotation.EntranceParserAutowiredAnnotation EntranceParser entranceParser) {
public ErrorCodeListener generateErrorCodeListener(
@PersistenceManagerBeanAnnotation.PersistenceManagerAutowiredAnnotation
PersistenceManager persistenceManager,
@EntranceParserBeanAnnotation.EntranceParserAutowiredAnnotation
EntranceParser entranceParser) {
PersistenceErrorCodeListener errorCodeListener = new PersistenceErrorCodeListener();
errorCodeListener.setEntranceParser(entranceParser);
errorCodeListener.setPersistenceManager(persistenceManager);
Expand All @@ -152,8 +163,8 @@ public ErrorCodeListener generateErrorCodeListener(@PersistenceManagerBeanAnnota
@ErrorCodeManagerBeanAnnotation
@ConditionalOnMissingBean(name = {ErrorCodeManagerBeanAnnotation.BEAN_NAME})
public ErrorCodeManager generateErrorCodeManager() {
/* try {
Class.forName("com.webank.wedatasphere.linkis.errorcode.client.handler.LinkisErrorCodeHandler");
/* try {
Class.forName("org.apache.linkis.errorcode.client.handler.LinkisErrorCodeHandler");
} catch (final Exception e) {
logger.error("failed to init linkis error code handler", e);
}*/
Expand All @@ -162,8 +173,11 @@ public ErrorCodeManager generateErrorCodeManager() {

@LogManagerBeanAnnotation
@ConditionalOnMissingBean(name = {LogManagerBeanAnnotation.BEAN_NAME})
public LogManager generateLogManager(@ErrorCodeListenerBeanAnnotation.ErrorCodeListenerAutowiredAnnotation ErrorCodeListener errorCodeListener,
@ErrorCodeManagerBeanAnnotation.ErrorCodeManagerAutowiredAnnotation ErrorCodeManager errorCodeManager){
public LogManager generateLogManager(
@ErrorCodeListenerBeanAnnotation.ErrorCodeListenerAutowiredAnnotation
ErrorCodeListener errorCodeListener,
@ErrorCodeManagerBeanAnnotation.ErrorCodeManagerAutowiredAnnotation
ErrorCodeManager errorCodeManager) {
CacheLogManager logManager = new CacheLogManager();
logManager.setErrorCodeListener(errorCodeListener);
logManager.setErrorCodeManager(errorCodeManager);
Expand All @@ -185,23 +199,30 @@ public ConsumerManager generateConsumerManager() {

@SchedulerContextBeanAnnotation
@ConditionalOnMissingBean(name = {SchedulerContextBeanAnnotation.BEAN_NAME})
public SchedulerContext generateSchedulerContext(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory,
@EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation ExecutorManager executorManager,
@ConsumerManagerBeanAnnotation.ConsumerManagerAutowiredAnnotation ConsumerManager consumerManager) {
public SchedulerContext generateSchedulerContext(
@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory,
@EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation
ExecutorManager executorManager,
@ConsumerManagerBeanAnnotation.ConsumerManagerAutowiredAnnotation
ConsumerManager consumerManager) {
return new EntranceSchedulerContext(groupFactory, consumerManager, executorManager);
}

@EntranceExecutorManagerBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceExecutorManagerBeanAnnotation.BEAN_NAME})
public ExecutorManager generateExecutorManager(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory) {
EngineConnManagerBuilder engineConnManagerBuilder = EngineConnManagerBuilder$.MODULE$.builder();
public ExecutorManager generateExecutorManager(
@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory) {
EngineConnManagerBuilder engineConnManagerBuilder =
EngineConnManagerBuilder$.MODULE$.builder();
engineConnManagerBuilder.setPolicy(Policy.Process);
return new EntranceExecutorManagerImpl(groupFactory, engineConnManagerBuilder.build());
}

@SchedulerBeanAnnotation
@ConditionalOnMissingBean(name = {SchedulerBeanAnnotation.BEAN_NAME})
public Scheduler generateScheduler(@SchedulerContextBeanAnnotation.SchedulerContextAutowiredAnnotation SchedulerContext schedulerContext) {
public Scheduler generateScheduler(
@SchedulerContextBeanAnnotation.SchedulerContextAutowiredAnnotation
SchedulerContext schedulerContext) {
Scheduler scheduler = new ParallelScheduler(schedulerContext);
scheduler.init();
scheduler.start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ public enum EntranceErrorCode {
INIT_JOB_ERROR(20012, "Init job error "),
RESULT_NOT_PERSISTED_ERROR(20013, "Result not persisted error "),
GROUP_NOT_FOUND(20014, "group not found"),
EXECUTION_CODE_ISNULL(20015, "execute code is null, nothing will be execute!(执行代码为空,没有任何代码会被执行)"),
EXECUTION_CODE_ISNULL(
20015, "execute code is null, nothing will be execute!(执行代码为空,没有任何代码会被执行)"),
JOB_UPDATE_FAILED(20016, "job update failed"),
VARIABLE_NULL_EXCEPTION(20017, "variable is null")
VARIABLE_NULL_EXCEPTION(20017, "variable is null");

;
private int errCode;
private String desc;

Expand Down
Loading

0 comments on commit 3a21bdc

Please sign in to comment.