Skip to content

Commit 8baae93

Browse files
dingxin-techdingxin-tech
committed
prepare for new version
1 parent ba600de commit 8baae93

File tree

14 files changed

+320
-55
lines changed

14 files changed

+320
-55
lines changed

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/ArrayRecord.java‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141

4242
import com.aliyun.odps.Column;
4343
import com.aliyun.odps.TableSchema;
44+
import com.aliyun.odps.exceptions.SchemaMismatchRuntimeException;
4445
import com.aliyun.odps.type.TypeInfoFactory;
4546

4647
/**
@@ -73,6 +74,13 @@ public class ArrayRecord implements Record, Serializable {
7374
*/
7475
private Long fieldMaxSize = DEFAULT_FIELD_MAX_SIZE;
7576

77+
/**
78+
* Whether to ignore case when accessing record by column name.
79+
* Since 0.51.8, default value from true to false.
80+
* if true, may cause performance loss.
81+
*/
82+
protected boolean caseSensitive;
83+
7684
public ArrayRecord(Column[] columns) {
7785
this(columns, true);
7886
}
@@ -82,31 +90,44 @@ public ArrayRecord(Column[] columns, boolean strictTypeValidation) {
8290
}
8391

8492
public ArrayRecord(Column[] columns, boolean strictTypeValidation, Long fieldMaxSize) {
93+
this(columns, strictTypeValidation, fieldMaxSize, false);
94+
}
95+
96+
public ArrayRecord(Column[] columns, boolean strictTypeValidation, Long fieldMaxSize, boolean caseSensitive) {
8597
if (columns == null) {
8698
throw new IllegalArgumentException();
8799
}
88100

89101
this.columns = columns;
90102
this.strictTypeValidation = strictTypeValidation;
103+
this.caseSensitive = caseSensitive;
91104
if (fieldMaxSize != null) {
92105
this.fieldMaxSize = fieldMaxSize;
93106
}
94107

95108
values = new Serializable[columns.length];
96109

97110
for (int i = 0; i < columns.length; i++) {
98-
nameMap.put(columns[i].getName().toLowerCase(), i);
111+
if (!caseSensitive) {
112+
nameMap.put(columns[i].getName().toLowerCase(), i);
113+
} else {
114+
nameMap.put(columns[i].getName(), i);
115+
}
99116
}
100117
}
101118

102119
public ArrayRecord(Column[] columns, Object[] values) {
103120
this(columns, values, true);
104121
}
105122

106-
public ArrayRecord(Column[] columns, Object[] values, boolean strictTypeValidation){
107-
this(columns, strictTypeValidation);
123+
public ArrayRecord(Column[] columns, Object[] values, boolean strictTypeValidation) {
124+
this(columns, values, strictTypeValidation, false);
125+
}
126+
127+
public ArrayRecord(Column[] columns, Object[] values, boolean strictTypeValidation, boolean caseSensitive) {
128+
this(columns, strictTypeValidation, null, caseSensitive);
108129
if (values.length != columns.length) {
109-
throw new IllegalArgumentException("Lengths of schema and column values of the Record mismatches.");
130+
throw new SchemaMismatchRuntimeException("Lengths of schema and column values of the Record mismatches.");
110131
}
111132
for (int i = 0 ; i < columns.length; i++){
112133
set(i, values[i]);
@@ -388,8 +409,11 @@ public Binary getBinary(String columnName) {
388409

389410
@Override
390411
public void set(Object[] values) {
391-
if (values == null || columns.length != values.length) {
392-
throw new IllegalArgumentException();
412+
if (values == null) {
413+
throw new IllegalArgumentException("values cannot be null.");
414+
}
415+
if (columns.length != values.length) {
416+
throw new SchemaMismatchRuntimeException("Schema mismatch, expect " + columns.length + " columns, but got " + values.length);
393417
}
394418
for (int i = 0; i < values.length; ++i) {
395419
set(i, values[i]);
@@ -402,7 +426,7 @@ public Object[] toArray() {
402426
}
403427

404428
private int getColumnIndex(String name) {
405-
Integer idx = nameMap.get(name.toLowerCase());
429+
Integer idx = nameMap.get(this.caseSensitive ? name : name.toLowerCase());
406430
if (idx == null) {
407431
throw new IllegalArgumentException("No such column:" + name);
408432
}

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/data/OdpsTypeTransformer.java‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.util.Map;
2121

2222
import com.aliyun.odps.OdpsType;
23+
import com.aliyun.odps.exceptions.SchemaMismatchRuntimeException;
2324
import com.aliyun.odps.type.ArrayTypeInfo;
2425
import com.aliyun.odps.type.CharTypeInfo;
2526
import com.aliyun.odps.type.DecimalTypeInfo;
@@ -95,8 +96,9 @@ private static void validateString(String value, long limit) {
9596
try {
9697
if (value.length() * UTF8_ENCODED_CHAR_MAX_SIZE > limit
9798
&& value.getBytes("utf-8").length > limit) {
98-
throw new IllegalArgumentException(
99-
"InvalidData: The string's length is more than " + limit + " bytes.");
99+
throw new SchemaMismatchRuntimeException(
100+
"InvalidData: The string's length is more than " + limit
101+
+ " bytes. You can enlarge this limit by 'setproject odps.sql.cfile2.field.maxsize=[length]' and rebuild the tunnel session or array record.");
100102
}
101103
} catch (UnsupportedEncodingException e) {
102104
throw new IllegalArgumentException(e.getMessage(), e);
@@ -113,23 +115,24 @@ private static void validateJson(Object value) {
113115

114116
private static void validateChar(Char value, CharTypeInfo typeInfo) {
115117
if (value.length() > typeInfo.getLength()) {
116-
throw new IllegalArgumentException(String.format(
118+
throw new SchemaMismatchRuntimeException(String.format(
117119
"InvalidData: %s data is overflow, pls check data length: %s.", typeInfo.getTypeName(),
118120
(value).length()));
119121
}
120122
}
121123

122124
private static void validateVarChar(Varchar value, VarcharTypeInfo typeInfo) {
123125
if (value.length() > typeInfo.getLength()) {
124-
throw new IllegalArgumentException(String.format(
126+
throw new SchemaMismatchRuntimeException(String.format(
125127
"InvalidData: %s data is overflow, pls check data length: %s.", typeInfo.getTypeName(),
126128
(value).length()));
127129
}
128130
}
129131

130132
private static void validateBigint(Long value) {
131133
if (value == Long.MIN_VALUE) {
132-
throw new IllegalArgumentException("InvalidData: Bigint out of range.");
134+
throw new IllegalArgumentException(
135+
"InvalidData: Bigint out of range. MaxCompute does not support Long type with a value of -2^63. For long numbers, you can use the String type instead.");
133136
}
134137
}
135138

@@ -143,15 +146,17 @@ private static void validateDateTime(ZonedDateTime value) {
143146

144147
private static void validateDateTime(long epochMilli) {
145148
if (epochMilli < DATETIME_MIN_TICKS || epochMilli > DATETIME_MAX_TICKS) {
146-
throw new IllegalArgumentException(String.format("InvalidData: Datetime(%s) out of range.", epochMilli));
149+
throw new IllegalArgumentException(String.format(
150+
"InvalidData: Datetime(%s) out of range. MaxCompute does not support time type before than -001-12-31 00:00:00 or"
151+
+ "after than 10000-01-02 00:00:00.", epochMilli));
147152
}
148153
}
149154

150155
private static void validateDecimal(BigDecimal value, DecimalTypeInfo typeInfo) {
151156
BigDecimal tmpValue = value.setScale(typeInfo.getScale(), RoundingMode.HALF_UP);
152157
int intLength = tmpValue.precision() - tmpValue.scale();
153158
if (intLength > (typeInfo.getPrecision() - typeInfo.getScale())) {
154-
throw new IllegalArgumentException(
159+
throw new SchemaMismatchRuntimeException(
155160
String.format("InvalidData: decimal value %s overflow, max integer digit number is %s.",
156161
value, (typeInfo.getPrecision() - typeInfo.getScale())));
157162
}
@@ -404,7 +409,7 @@ static <T> T transform(Object value,
404409
transformedResult);
405410
} catch (ClassCastException e) {
406411
// manually throw exception because jvm may optimize ClassCastException message to null.
407-
throw new IllegalArgumentException("Cannot format " + value
412+
throw new SchemaMismatchRuntimeException("Cannot format " + value
408413
+ "(" + value.getClass() + ") to ODPS type: "
409414
+ typeInfo.getOdpsType()
410415
+ ", expect java class: "

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/exceptions/SchemaMismatchException.java‎ renamed to ‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/exceptions/SchemaMismatchException.java‎

File renamed without changes.

‎odps-sdk/odps-sdk-commons/src/main/java/com/aliyun/odps/exceptions/SchemaMismatchRuntimeException.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package com.aliyun.odps.exceptions;
2+
3+
/**
4+
* @author dingxin ([email protected])
5+
* <p>
6+
* client side schema mismatch exception.
7+
* only throw at when input value of record is not match with table schema.
8+
*/
9+
public class SchemaMismatchRuntimeException extends IllegalArgumentException {
10+
11+
public SchemaMismatchRuntimeException(String errorMsg) {
12+
super(errorMsg);
13+
}
14+
15+
public SchemaMismatchRuntimeException(String errorMsg, Exception e) {
16+
super(errorMsg, e);
17+
}
18+
}

‎odps-sdk/odps-sdk-core/src/main/java/com/aliyun/odps/LogView.java‎

Lines changed: 96 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,36 +22,55 @@
2222
import java.util.HashMap;
2323

2424
import com.aliyun.odps.commons.transport.Response;
25+
import com.aliyun.odps.options.OdpsOptions;
2526
import com.aliyun.odps.rest.RestClient;
2627
import com.aliyun.odps.security.SecurityManager;
2728
import com.aliyun.odps.utils.StringUtils;
2829

30+
/**
31+
* JobInsight is the name of new version of Logview.
32+
* In MaxCompute S54, we will use JobInsight to replace Logview.
33+
*
34+
*/
2935
public class LogView {
3036

3137
private static final String POLICY_TYPE = "BEARER";
32-
private static final String HOST_DEFAULT = "http://logview.aliyun.com";
38+
private static final String LOGVIEW_HOST_DEFAULT = "http://logview.alibaba-inc.com";
3339
private String logViewHost = "";
3440

35-
private static final String HOST_DEFAULT_V2 = "https://maxcompute.console.aliyun.com";
41+
private static final String JOBINSIGHT_HOST_DEFAULT = "https://maxcompute.console.aliyun.com";
42+
private String jobInsightHost = "";
3643

37-
private int version = 1;
44+
private final int version;
3845

3946
Odps odps;
4047

4148
public LogView(Odps odps) {
42-
this(odps, 1);
49+
this(odps, null);
4350
}
4451

45-
public LogView(Odps odps, int version) {
52+
public LogView(Odps odps, Integer version) {
4653
this.odps = odps;
47-
this.version = version;
54+
if (version == null) {
55+
Boolean useLegacyLogview = odps.options().isUseLegacyLogview();
56+
if (useLegacyLogview == null) {
57+
this.jobInsightHost = getJobInsightHost();
58+
if (StringUtils.isNullOrEmpty(jobInsightHost)) {
59+
this.version = 1;
60+
} else {
61+
this.version = 2;
62+
}
63+
} else if (useLegacyLogview) {
64+
this.version = 1;
65+
} else {
66+
this.version = 2;
67+
}
68+
} else {
69+
this.version = version;
70+
}
4871
}
4972

5073
private String getLogviewHost() {
51-
if (2 == version) {
52-
return HOST_DEFAULT_V2;
53-
}
54-
5574
if (odps.getLogViewHost() != null) {
5675
return odps.getLogViewHost();
5776
} else {
@@ -62,13 +81,32 @@ private String getLogviewHost() {
6281
Response resp = restClient.request(resource, "GET", params, null, null);
6382
String logViewHost = new String(resp.getBody());
6483
if (StringUtils.isNullOrEmpty(logViewHost)) {
65-
return HOST_DEFAULT;
84+
return LOGVIEW_HOST_DEFAULT;
6685
} else
6786
{
6887
return logViewHost;
6988
}
7089
} catch (Exception e) {
71-
return HOST_DEFAULT;
90+
return LOGVIEW_HOST_DEFAULT;
91+
}
92+
}
93+
}
94+
95+
96+
private String getJobInsightHost() {
97+
if (odps.getJobInsightHost() != null) {
98+
return odps.getJobInsightHost();
99+
} else {
100+
RestClient restClient = odps.clone().getRestClient();
101+
restClient.setConnectTimeout(3);
102+
restClient.setReadTimeout(10);
103+
try {
104+
String resource = "/webconsole/host";
105+
HashMap<String, String> params = new HashMap<String, String>();
106+
Response resp = restClient.request(resource, "GET", params, null, null);
107+
return new String(resp.getBody());
108+
} catch (Exception e) {
109+
return null;
72110
}
73111
}
74112
}
@@ -79,11 +117,17 @@ private String getLogviewHost() {
79117
* @return logview host 地址
80118
*/
81119
public String getLogViewHost() {
82-
if (StringUtils.isNullOrEmpty(logViewHost)) {
83-
logViewHost = getLogviewHost();
120+
if (version == 2) {
121+
if (StringUtils.isNullOrEmpty(jobInsightHost)) {
122+
jobInsightHost = getJobInsightHost();
123+
}
124+
return jobInsightHost;
125+
} else {
126+
if (StringUtils.isNullOrEmpty(logViewHost)) {
127+
logViewHost = getLogviewHost();
128+
}
129+
return logViewHost;
84130
}
85-
86-
return logViewHost;
87131
}
88132

89133
/**
@@ -95,6 +139,27 @@ public void setLogViewHost(String logViewHost) {
95139
this.logViewHost = logViewHost;
96140
}
97141

142+
/**
143+
* 设置 logview host 地址
144+
* @param jobInsightHost
145+
* host 地址
146+
*/
147+
public void setJobInsightHost(String jobInsightHost) {
148+
this.jobInsightHost = jobInsightHost;
149+
}
150+
151+
/**
152+
* 生成 logview 链接，默认有效时间 24h
153+
*
154+
* @param instance
155+
* instance 对象
156+
* @return logview
157+
* @throws OdpsException
158+
*/
159+
public String generateLogView(Instance instance) throws OdpsException {
160+
return generateLogView(instance, 24, null, null);
161+
}
162+
98163
/**
99164
* 生成 logview 链接
100165
*
@@ -143,11 +208,10 @@ public String generateSubQueryLogView(Instance instance, int queryId, String tok
143208

144209
private String generateLogView(Instance instance, long hours, Integer queryId, String token)
145210
throws OdpsException {
146-
if (StringUtils.isNullOrEmpty(logViewHost)) {
147-
logViewHost = getLogviewHost();
148-
}
149-
150211
if (1 == version) {
212+
if (StringUtils.isNullOrEmpty(logViewHost)) {
213+
logViewHost = getLogviewHost();
214+
}
151215
StringBuilder urlBuilder = new StringBuilder(logViewHost);
152216
urlBuilder.append("/logview/?h=").append(odps.getEndpoint())
153217
.append("&p=").append(instance.getProject())
@@ -161,10 +225,18 @@ private String generateLogView(Instance instance, long hours, Integer queryId, S
161225
urlBuilder.append("&token=").append(token);
162226
return urlBuilder.toString();
163227
} else if (2 == version) {
164-
String url = logViewHost + "/" + odps.projects().get().getRegionId()
165-
+ "/job-insights?h=" + odps.getEndpoint()
166-
+ "&p=" + instance.getProject()
167-
+ "&i=" + instance.getId();
228+
if (StringUtils.isNullOrEmpty(jobInsightHost)) {
229+
jobInsightHost = getJobInsightHost();
230+
}
231+
String regionId = "cn";
232+
try {
233+
regionId = odps.projects().get(instance.getProject()).getRegionId();
234+
} catch (Exception ignore) {
235+
}
236+
String url = jobInsightHost + "/" + regionId
237+
+ "/job-insights?h=" + odps.getEndpoint()
238+
+ "&p=" + instance.getProject()
239+
+ "&i=" + instance.getId();
168240
if (queryId != null) {
169241
url += "&subQuery=" + queryId;
170242
}

0 commit comments

Comments
 (0)