Skip to content

Commit 1f98595

Browse files
baiyubin2020baiyubin2020
authored
Merge pull request #56 from aliyun/fix-image-mirror
Fix image mirror replication
2 parents 091405f + 6a3091f commit 1f98595

29 files changed

+1064
-122
lines changed

‎src/main/java/com/aliyun/oss/InconsistentException.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package com.aliyun.oss;
21+
22+
/**
23+
* <p>
24+
* 表示OSS端的数据与SDK端的数据不一致。
25+
* </p>
26+
*
27+
*
28+
* <p>
29+
* 通常来讲，调用者需要处理{@link InconsistentException}。因为该异常表明请求被服务处理，
30+
* 上传操作已经成功，但是OSS端的数据与SDK端不一致。调用者需要上传上传的文件然后重新删除。
31+
* </p>
32+
*
33+
* <p>
34+
* 抛出该异常的操作包括putObject、appendObject、uploadPart、uploadFile等上传操作，
35+
* getObject下载操作的数据一致性，调用者需要在数据流读取结束后校验，即比较OSS端与SDK端的数据校验和。
36+
* </p>
37+
*
38+
*/
39+
public class InconsistentException extends RuntimeException {
40+
41+
private static final long serialVersionUID = 2140587868503948665L;
42+
43+
private Long clientChecksum;
44+
private Long serverChecksum;
45+
private String requestId;
46+
47+
public InconsistentException(Long clientChecksum, Long serverChecksum, String requestId) {
48+
super();
49+
this.clientChecksum = clientChecksum;
50+
this.serverChecksum = serverChecksum;
51+
this.requestId = requestId;
52+
}
53+
54+
public Long getClientChecksum() {
55+
return clientChecksum;
56+
}
57+
58+
public void setClientChecksum(Long clientChecksum) {
59+
this.clientChecksum = clientChecksum;
60+
}
61+
62+
public Long getServerChecksum() {
63+
return serverChecksum;
64+
}
65+
66+
public void setServerChecksum(Long serverChecksum) {
67+
this.serverChecksum = serverChecksum;
68+
}
69+
70+
public String getRequestId() {
71+
return requestId;
72+
}
73+
74+
public void setRequestId(String requestId) {
75+
this.requestId = requestId;
76+
}
77+
78+
@Override
79+
public String getMessage() {
80+
return "InconsistentException " + "\n[RequestId]: " + getRequestId()
81+
+ "\n[ClientChecksum]: " + getClientChecksum()
82+
+ "\n[ServerChecksum]: " + getServerChecksum();
83+
}
84+
85+
}

‎src/main/java/com/aliyun/oss/OSSClient.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
import com.aliyun.oss.internal.OSSObjectOperation;
7272
import com.aliyun.oss.internal.OSSUploadOperation;
7373
import com.aliyun.oss.internal.OSSUtils;
74+
import com.aliyun.oss.internal.RequestParameters;
7475
import com.aliyun.oss.internal.SignUtils;
7576
import com.aliyun.oss.model.AbortMultipartUploadRequest;
7677
import com.aliyun.oss.model.AccessControlList;
@@ -786,10 +787,10 @@ public URL generatePresignedUrl(GeneratePresignedUrlRequest request)
786787
requestMessage.setResourcePath(resourcePath);
787788

788789
requestMessage.addHeader(HttpHeaders.DATE, expires);
789-
if (request.getContentType() != null && request.getContentType().trim() != "") {
790+
if (request.getContentType() != null && !request.getContentType().trim().equals("")) {
790791
requestMessage.addHeader(HttpHeaders.CONTENT_TYPE, request.getContentType());
791792
}
792-
if (request.getContentMD5() != null && request.getContentMD5().trim() != "") {
793+
if (request.getContentMD5() != null && request.getContentMD5().trim().equals("")) {
793794
requestMessage.addHeader(HttpHeaders.CONTENT_MD5, request.getContentMD5());
794795
}
795796
for (Map.Entry<String, String> h : request.getUserMetadata().entrySet()) {
@@ -808,6 +809,10 @@ public URL generatePresignedUrl(GeneratePresignedUrlRequest request)
808809
}
809810
}
810811

812+
if (request.getProcess() != null && !request.getProcess().trim().equals("")) {
813+
requestMessage.addParameter(RequestParameters.SUBRESOURCE_PROCESS, request.getProcess());
814+
}
815+
811816
if (useSecurityToken) {
812817
requestMessage.addParameter(SECURITY_TOKEN, currentCreds.getSecurityToken());
813818
}

‎src/main/java/com/aliyun/oss/common/parser/RequestMarshallers.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.aliyun.oss.ClientException;
3131
import com.aliyun.oss.common.comm.io.FixedLengthInputStream;
3232
import com.aliyun.oss.common.utils.DateUtil;
33+
import com.aliyun.oss.model.AddBucketReplicationRequest.ReplicationAction;
3334
import com.aliyun.oss.model.BucketReferer;
3435
import com.aliyun.oss.model.CompleteMultipartUploadRequest;
3536
import com.aliyun.oss.model.CreateBucketRequest;
@@ -315,6 +316,18 @@ public FixedLengthInputStream marshall(SetBucketWebsiteRequest request) {
315316
if (redirect.getMirrorURL() != null) {
316317
xmlBody.append("<MirrorURL>" + redirect.getMirrorURL() + "</MirrorURL>");
317318
}
319+
if (redirect.getMirrorSecondaryURL() != null) {
320+
xmlBody.append("<MirrorURLSlave>" + redirect.getMirrorSecondaryURL() + "</MirrorURLSlave>");
321+
}
322+
if (redirect.getMirrorProbeURL() != null) {
323+
xmlBody.append("<MirrorURLProbe>" + redirect.getMirrorProbeURL() + "</MirrorURLProbe>");
324+
}
325+
if (redirect.isPassQueryString() != null) {
326+
xmlBody.append("<MirrorPassQueryString>" + redirect.isPassQueryString() + "</MirrorPassQueryString>");
327+
}
328+
if (redirect.isPassOriginalSlashes() != null) {
329+
xmlBody.append("<MirrorPassOriginalSlashes>" + redirect.isPassOriginalSlashes() + "</MirrorPassOriginalSlashes>");
330+
}
318331
xmlBody.append("</Redirect>");
319332
xmlBody.append("</RoutingRule>");
320333
}
@@ -510,6 +523,17 @@ public FixedLengthInputStream marshall(AddBucketReplicationRequest request) {
510523
} else {
511524
xmlBody.append("<HistoricalObjectReplication>" + "disabled" + "</HistoricalObjectReplication>");
512525
}
526+
if (request.getObjectPrefixList() != null && request.getObjectPrefixList().size() > 0) {
527+
xmlBody.append("<PrefixSet>");
528+
for (String prefix : request.getObjectPrefixList()) {
529+
xmlBody.append("<Prefix>" + prefix + "</Prefix>");
530+
}
531+
xmlBody.append("</PrefixSet>");
532+
}
533+
if (request.getReplicationActionList() != null && request.getReplicationActionList().size() > 0) {
534+
xmlBody.append("<Action>" + RequestMarshallers.joinRepliationAction(
535+
request.getReplicationActionList()) + "</Action>");
536+
}
513537
xmlBody.append("</Rule>");
514538
xmlBody.append("</ReplicationConfiguration>");
515539
return stringMarshaller.marshall(xmlBody.toString());
@@ -717,4 +741,20 @@ private static String escapeKey(String key) {
717741
return builder.toString();
718742
}
719743

744+
private static String joinRepliationAction(List <ReplicationAction> actions) {
745+
StringBuilder sb = new StringBuilder();
746+
boolean first = true;
747+
748+
for (ReplicationAction action : actions) {
749+
if (!first) {
750+
sb.append(",");
751+
}
752+
sb.append(action);
753+
754+
first = false;
755+
}
756+
757+
return sb.toString();
758+
}
759+
720760
}

‎src/main/java/com/aliyun/oss/common/utils/IOUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ public static InputStream newRepeatableInputStream(final BoundedInputStream orig
154154
return repeatable;
155155
}
156156

157-
public static Long getCRC64Value(InputStream inputStream) {
157+
public static Long getCRCValue(InputStream inputStream) {
158158
if (inputStream instanceof CheckedInputStream) {
159159
return ((CheckedInputStream) inputStream).getChecksum().getValue();
160160
}

‎src/main/java/com/aliyun/oss/internal/OSSMultipartOperation.java‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,12 @@ public int compare(PartETag p1, PartETag p2) {
186186
} else {
187187
result = doOperation(request, completeMultipartUploadCallbackResponseParser, bucketName, key, true, null, reponseHandlers);
188188
}
189-
result.setClientCRC64(calcObjectCRC64FromParts(completeMultipartUploadRequest.getPartETags()));
189+
190+
result.setClientCRC(calcObjectCRCFromParts(completeMultipartUploadRequest.getPartETags()));
191+
if (getInnerClient().getClientConfiguration().isCrcCheckEnabled()) {
192+
OSSUtils.checkChecksum(result.getClientCRC(), result.getServerCRC(), result.getRequestId());
193+
}
194+
190195
return result;
191196
}
192197

@@ -372,7 +377,12 @@ public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
372377
result.setETag(trimQuotes(response.getHeaders().get(OSSHeaders.ETAG)));
373378
result.setRequestId(response.getRequestId());
374379
result.setPartSize(uploadPartRequest.getPartSize());
375-
ResponseParsers.setCRC64(result, response);
380+
ResponseParsers.setCRC(result, response);
381+
382+
if (getInnerClient().getClientConfiguration().isCrcCheckEnabled()) {
383+
OSSUtils.checkChecksum(result.getClientCRC(), result.getServerCRC(), result.getRequestId());
384+
}
385+
376386
return result;
377387
}
378388

@@ -550,15 +560,15 @@ private static void populateCompleteMultipartUploadOptionalHeaders(
550560
}
551561
}
552562

553-
private static Long calcObjectCRC64FromParts(List<PartETag> partETags) {
554-
Long crc = 0L;
563+
private static Long calcObjectCRCFromParts(List<PartETag> partETags) {
564+
long crc = 0;
555565
for (PartETag partETag : partETags) {
556-
if (partETag.getPartCRC64() == null) {
566+
if (partETag.getPartCRC() == null || partETag.getPartSize() <= 0) {
557567
return null;
558568
}
559-
crc = CRC64.combine(crc, partETag.getPartCRC64(), partETag.getPartSize());
569+
crc = CRC64.combine(crc, partETag.getPartCRC(), partETag.getPartSize());
560570
}
561-
return crc;
571+
return new Long(crc);
562572
}
563573

564574
}

‎src/main/java/com/aliyun/oss/internal/OSSObjectOperation.java‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,12 +135,22 @@ public OSSObjectOperation(ServiceClient client, CredentialsProvider credsProvide
135135
*/
136136
public PutObjectResult putObject(PutObjectRequest putObjectRequest)
137137
throws OSSException, ClientException {
138+
138139
assertParameterNotNull(putObjectRequest, "putObjectRequest");
140+
141+
PutObjectResult result = null;
142+
139143
if (putObjectRequest.getCallback() == null) {
140-
return writeObjectInternal(WriteMode.OVERWRITE, putObjectRequest, putObjectReponseParser);
144+
result = writeObjectInternal(WriteMode.OVERWRITE, putObjectRequest, putObjectReponseParser);
141145
} else {
142-
return writeObjectInternal(WriteMode.OVERWRITE, putObjectRequest, putObjectCallbackReponseParser);
146+
result = writeObjectInternal(WriteMode.OVERWRITE, putObjectRequest, putObjectCallbackReponseParser);
147+
}
148+
149+
if (getInnerClient().getClientConfiguration().isCrcCheckEnabled()) {
150+
OSSUtils.checkChecksum(result.getClientCRC(), result.getServerCRC(), result.getRequestId());
143151
}
152+
153+
return result;
144154
}
145155

146156
/**
@@ -173,12 +183,21 @@ public PutObjectResult putObject(URL signedUrl, InputStream requestContent, long
173183
*/
174184
public AppendObjectResult appendObject(AppendObjectRequest appendObjectRequest)
175185
throws OSSException, ClientException {
186+
176187
assertParameterNotNull(appendObjectRequest, "appendObjectRequest");
188+
177189
AppendObjectResult result = writeObjectInternal(WriteMode.APPEND, appendObjectRequest, appendObjectResponseParser);
178-
if (appendObjectRequest.getPreviousCRC64() != null && result.getClientCRC64() != null) {
179-
result.setClientCRC64(CRC64.combine(appendObjectRequest.getPreviousCRC64(), result.getClientCRC64(),
190+
191+
if (appendObjectRequest.getInitCRC() != null && result.getClientCRC() != null) {
192+
result.setClientCRC(CRC64.combine(appendObjectRequest.getInitCRC(), result.getClientCRC(),
180193
(result.getNextPosition() - appendObjectRequest.getPosition())));
181194
}
195+
196+
if (getInnerClient().getClientConfiguration().isCrcCheckEnabled() &&
197+
appendObjectRequest.getInitCRC() != null) {
198+
OSSUtils.checkChecksum(result.getClientCRC(), result.getServerCRC(), result.getRequestId());
199+
}
200+
182201
return result;
183202
}
184203

@@ -211,6 +230,11 @@ public OSSObject getObject(GetObjectRequest getObjectRequest)
211230
Map<String, String> params = new HashMap<String, String>();
212231
populateResponseHeaderParameters(params, getObjectRequest.getResponseHeaders());
213232

233+
String process = getObjectRequest.getProcess();
234+
if (process != null) {
235+
params.put(RequestParameters.SUBRESOURCE_PROCESS, process);
236+
}
237+
214238
request = new OSSRequestMessageBuilder(getInnerClient())
215239
.setEndpoint(getEndpoint())
216240
.setMethod(HttpMethod.GET)

‎src/main/java/com/aliyun/oss/internal/OSSUtils.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.util.Map.Entry;
3838

3939
import com.aliyun.oss.ClientConfiguration;
40+
import com.aliyun.oss.InconsistentException;
4041
import com.aliyun.oss.common.comm.ResponseMessage;
4142
import com.aliyun.oss.common.utils.BinaryUtil;
4243
import com.aliyun.oss.common.utils.CodingUtils;
@@ -470,4 +471,14 @@ public static void populateRequestCallback(Map<String, String> headers, Callback
470471
}
471472
}
472473

474+
/**
475+
* 检测OSS和SDK计算的校验和是否相同，不同抛异常InconsistentException
476+
*/
477+
public static void checkChecksum(Long clientChecksum, Long serverChecksum, String requestId) {
478+
if (clientChecksum != null && serverChecksum != null &&
479+
!clientChecksum.equals(serverChecksum)) {
480+
throw new InconsistentException(clientChecksum, serverChecksum, requestId);
481+
}
482+
}
483+
473484
}

0 commit comments

Comments
 (0)