Skip to content
Open

1 #325

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions EasyExcel/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.netease.lowcode.extensions</groupId>
<artifactId>EasyExcel</artifactId>
<version>2.4.2</version>
<version>2.5.0</version>

<properties>
<maven.compiler.source>8</maven.compiler.source>
Expand Down Expand Up @@ -122,7 +122,7 @@
<version>1.7.1</version>
<configuration>
<jarWithDependencies>false</jarWithDependencies>
<!-- <rewriteVersion>true</rewriteVersion>-->
<rewriteVersion>true</rewriteVersion>
</configuration>
<executions>
<execution>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ public static ExcelResponse createExcelWithUrl(CustomExcelData excelData) {
}

/**
* [推荐] 解析excel
* [推荐] 解析excel-数据传入handle
*
* @param handle
* @param request
Expand Down Expand Up @@ -474,6 +474,48 @@ public static ParseBigDataResponse parseBigDataExcel(Function<List<String>, Stri
return ParseBigDataResponse.OK(readListener.getTotal());
}


/**
* [推荐] 解析excel-数据直接返回
*
* @param request
* @return
*/
@NaslLogic
public static ParseBigDataResponse parseBigDataExcelNoSave(ParseRequest request) {

if (Objects.isNull(request) || StringUtils.isBlank(request.fullClassName)) {
return ParseBigDataResponse.FAIL("请求参数为空,fullClassName不能为空!");
}
if (StringUtils.isBlank(request.url)) {
return ParseBigDataResponse.FAIL("文件url不能为空!");
}
Class<?> clazz;
try {
clazz = Class.forName(request.getFullClassName(), false, Thread.currentThread().getContextClassLoader());
} catch (ClassNotFoundException e) {
log.error("加载类失败", e);
return ParseBigDataResponse.FAIL(String.format("加载类[%s]失败!", request.getFullClassName()), e.getMessage() + Arrays.toString(e.getStackTrace()));
}

LibraryReadListener<?> readListener = new LibraryReadListener<>(null, clazz, request, fileUtils);
SimpleReadCacheSelector simpleReadCacheSelector = new SimpleReadCacheSelector();
// 放多少批数据在内存,默认20
if (Objects.nonNull(request.getMaxCacheActivateBatchCount())) {
simpleReadCacheSelector.setMaxCacheActivateBatchCount(request.getMaxCacheActivateBatchCount());
}
try (InputStream inputStream = openUrlStream(request.getUrl())) {
EasyExcel.read(inputStream, clazz, readListener).readCacheSelector(simpleReadCacheSelector).sheet().doRead();
} catch (RuntimeException e) {
log.error("excel解析失败", e);
return ParseBigDataResponse.FAIL("excel解析失败!", e.getMessage() + Arrays.toString(e.getStackTrace()));
} catch (IOException e) {
log.error("文件下载失败", e);
return ParseBigDataResponse.FAIL("文件下载失败", e.getMessage() + Arrays.toString(e.getStackTrace()));
}
return ParseBigDataResponse.OK(readListener.getTotal(), readListener.getCachedDataList());
}

/**
* [推荐使用]导出大数据量excel
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public LibraryReadListener(Function<List<String>, String> handle, Class<T> type,
}


private List<String> toJsonString(List<T> data) {
public List<String> toJsonString(List<T> data) {
// https://blog.csdn.net/fenglllle/article/details/114293427
// 使用fastjson2会出现classloader问题(cast to exception)

Expand Down Expand Up @@ -131,8 +131,10 @@ public void invoke(T data, AnalysisContext context) {
// 解析到一条数据,加入缓存
cachedDataList.add(data);
if (cachedDataList.size() >= BATCH_COUNT) {
// 调用function处理数据
handle.apply(toJsonString(cachedDataList));
if (handle != null) {
// 调用function处理数据
handle.apply(toJsonString(cachedDataList));
}
// 处理完清理list
cachedDataList = ListUtils.newArrayListWithExpectedSize(BATCH_COUNT);
}
Expand Down Expand Up @@ -216,8 +218,10 @@ private void fillPicture(T data, AnalysisContext context) {

@Override
public void doAfterAllAnalysed(AnalysisContext context) {
// 调用function处理数据
handle.apply(toJsonString(cachedDataList));
if (handle != null) {
// 调用function处理数据
handle.apply(toJsonString(cachedDataList));
}
// 数据处理完成
}

Expand All @@ -241,6 +245,15 @@ public boolean hasNext(AnalysisContext context) {
return ReadListener.super.hasNext(context);
}

/**
* 获取缓存数据
*
* @return
*/
public List<String> getCachedDataList() {
return toJsonString(cachedDataList);
}

/**
* 获取导入数量
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import com.netease.lowcode.core.annotation.NaslStructure;

import java.util.List;

@NaslStructure
public class ParseBigDataResponse {

Expand All @@ -12,12 +14,16 @@ public class ParseBigDataResponse {
public Double cost;// 处理耗时,单位 s
public Double size;// 文件大小,单位字节
public Long total;// 统计导入条数
/**
* 数据列表
*/
public List<String> dataList;

public static ParseBigDataResponse FAIL(String msg) {
return FAIL(msg,null);
return FAIL(msg, null);
}

public static ParseBigDataResponse FAIL(String msg,String trace){
public static ParseBigDataResponse FAIL(String msg, String trace) {
ParseBigDataResponse fail = new ParseBigDataResponse();
fail.setSuccess(false);
fail.setMsg(msg);
Expand All @@ -29,6 +35,13 @@ public static ParseBigDataResponse OK() {
return OK("success");
}

public static ParseBigDataResponse OK(Long total, List<String> dataList) {
ParseBigDataResponse ok = OK();
ok.setTotal(total);
ok.setDataList(dataList);
return ok;
}

public static ParseBigDataResponse OK(Long total) {
ParseBigDataResponse ok = OK();
ok.setTotal(total);
Expand All @@ -42,6 +55,10 @@ public static ParseBigDataResponse OK(String msg) {
return ok;
}

public void setDataList(List<String> dataList) {
this.dataList = dataList;
}

public Boolean getSuccess() {
return success;
}
Expand Down
5 changes: 3 additions & 2 deletions async_function_pool/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

<groupId>com.netease</groupId>
<artifactId>async_function_pool</artifactId>
<version>1.1.0</version>
<version>1.3.2</version>

<properties>
<maven.compiler.source>8</maven.compiler.source>
Expand All @@ -39,6 +39,7 @@
<artifactId>lcap-annotation</artifactId>
<groupId>com.netease.lowcode</groupId>
<version>1.0.4</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
Expand Down Expand Up @@ -75,7 +76,7 @@
<version>1.4.3</version>
<configuration>
<jarWithDependencies>false</jarWithDependencies>
<rewriteVersion>true</rewriteVersion>
<!-- <rewriteVersion>true</rewriteVersion>-->
</configuration>
<executions>
<execution>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
@Component
public class FunctionManagerApi {

private static final Logger logger = LoggerFactory.getLogger("LCAP_CUSTOMIZE_LOGGER");
private static final Logger logger = LoggerFactory.getLogger("LCAP_EXTENSION_LOGGER");

/**
* 逻辑关键词,逻辑
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executor;

import static java.lang.Thread.sleep;

@Aspect
@Component
public class ExtensionAsyncLogicExcuteAspect {
Expand All @@ -37,7 +35,6 @@ public Object aroundCustomizeServiceMethods(ProceedingJoinPoint joinPoint) throw
}
CompletableFuture<Object> future = CompletableFuture.supplyAsync(() -> {
try {
sleep(10*1000);
return joinPoint.proceed();
} catch (Throwable e) {
log.error("ExtensionAsyncLogicExcuteAspect error", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.slf4j.LoggerFactory;
import org.springframework.core.io.ClassPathResource;
import org.springframework.util.FileCopyUtils;
import org.springframework.util.StringUtils;

import java.io.InputStream;
import java.nio.charset.StandardCharsets;
Expand All @@ -28,7 +29,7 @@ public static JSONArray readAnnotationFile(String annoName) {
JSONArray array = JSONObject.parseArray(new String(bdata, StandardCharsets.UTF_8));
return array;
} catch (Exception e) {
log.error("readAnnotationFile error", e);
log.warn("readAnnotationFile error", e);
// 处理异常
return null;
}
Expand All @@ -51,8 +52,9 @@ public static List<String> listAlluseAnnoLogicNames(String annoName) {
JSONObject annoObj = (JSONObject) anno;
String logicName = annoObj.getString("logicName");
JSONObject annotationProperties = annoObj.getJSONObject("annotationProperties");
if ("true".equals(annotationProperties.getString("useAnno"))) {
if ("true".equals(annotationProperties.getString("useAnno")) && !StringUtils.isEmpty(logicName)) {
result.add(logicName);
result.add(NamingUtils.toLowerCamelCase(logicName));
}
});
useAnnoLogicNames.put(annoName, result);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package com.netease.lib.tasks.util;

public class NamingUtils {
public static String toLowerCamelCase(String str) {
if (str == null || str.isEmpty()) {
return str;
}

// 如果已经是首字母小写,直接返回
if (Character.isLowerCase(str.charAt(0))) {
return str;
}

// 处理多单词的驼峰命名
StringBuilder result = new StringBuilder();
boolean firstWord = true;

// 分割单词(根据大写字母分割)
for (int i = 0; i < str.length(); i++) {
char c = str.charAt(i);
if (Character.isUpperCase(c) && i > 0) {
if (firstWord) {
result.append(Character.toLowerCase(c));
firstWord = false;
} else {
result.append(c);
}
} else {
result.append(firstWord ? Character.toLowerCase(c) : c);
firstWord = false;
}
}

return result.toString();
}
}
2 changes: 1 addition & 1 deletion excel-parser/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

<groupId>com.netease.lowcode.extension</groupId>
<artifactId>excel-parser</artifactId>
<version>1.3.0</version>
<version>1.3.1</version>
<name>extension module archetype</name>
<packaging>jar</packaging>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
import org.springframework.web.context.request.ServletRequestAttributes;

import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
Expand All @@ -44,8 +45,8 @@ public class ExcelParser {
* @param columnFieldMap 列映射:key表示列的索引,value表示映射的字段名称
* @param row 表头的行数
* @param clazz 映射类
* @return
* @param <T>
* @return
*/
public static <T> ExcelParseResult<T> parseAllSheet(String url, ExcelParseRect rect, Map<String, String> columnFieldMap, Long row, Class<T> clazz) {
return parseBySheetName(url, null, rect, columnFieldMap, row, clazz);
Expand Down Expand Up @@ -161,24 +162,25 @@ private static InputStream openUrlStream(String url) {

/**
* 获取文件下载路径,因为是请求本服务所以替换域名为localhost避免出现一些hostname unknown的情况
*
* @param url
* @return
*/
private static String getFileUrl(String url) {
int index = url.indexOf("/upload/");
HttpServletRequest request = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest();
int port = request.getLocalPort();
String contextPath = request.getContextPath();

if (index >= 0) {
//制品中上传文件后的url
url = "http://localhost:" + port + "/upload/" + url.substring(index + "/upload/".length());

if (StringUtils.isNotBlank(contextPath) && !"/".equals(contextPath)) {
url = "http://localhost:" + port + contextPath + "/upload/" + url.substring(index + "/upload/".length());
ServletRequestAttributes servletRequestAttributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
if (servletRequestAttributes != null) {
HttpServletRequest request = servletRequestAttributes.getRequest();
int port = request.getLocalPort();
String contextPath = request.getContextPath();
if (index >= 0) {
//制品中上传文件后的url
url = "http://localhost:" + port + "/upload/" + url.substring(index + "/upload/".length());
if (StringUtils.isNotBlank(contextPath) && !"/".equals(contextPath)) {
url = "http://localhost:" + port + contextPath + "/upload/" + url.substring(index + "/upload/".length());
}
}
}

log.info("文件下载路径{}", url);
return url;
}
Expand Down
2 changes: 1 addition & 1 deletion httpclient/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<artifactId>httpclient</artifactId>
<name>httpclient</name>
<description>httpclient</description>
<version>1.4.1</version>
<version>1.4.2</version>

<dependencies>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ public void setExpandLogicAuthFlag(String expandLogicAuthFlag) {
@Override
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
if ("0".equals(this.getExpandLogicAuthFlag())) {
log.debug("逻辑鉴权开关关闭");
filterChain.doFilter(servletRequest, servletResponse);
return;
}
Expand All @@ -80,7 +79,6 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo
String requestURI = httpRequest.getRequestURI();
String method = httpRequest.getMethod();
String logicIdentifier = requestURI + LOGIC_IDENTIFIER_SEPARATOR + method;
log.info("当前请求的逻辑标识: {}", logicIdentifier);
try {
if ("1".equals(this.getExpandLogicAuthFlag()) || "3".equals(this.getExpandLogicAuthFlag())) {
Map userInfo = getSession(httpRequest);
Expand Down