Skip to content

Commit

Permalink
Scripting: augmented javadoc to api spec (#69082) (#69138)
Browse files Browse the repository at this point in the history 
Accept system property `packageSources` with mapping of
package names to source roots.

Format `<package0>:<path0>;<package1>:<path1>`.

Checks ESv2 License in addition to GPLv2.

Backport: dde3df2
  • Loading branch information
@stu-elastic
stu-elastic authored Feb 17, 2021
1 parent b552495 commit 16a1662
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 37 deletions.
1 change: 1 addition & 0 deletions modules/lang-painless/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ tasks.register("generateContextApiSpec", DefaultTestClustersTask) {
classpath = sourceSets.doc.runtimeClasspath
systemProperty "cluster.uri", "${-> testClusters.generateContextApiSpecCluster.singleNode().getAllHttpSocketURI().get(0)}"
systemProperty "jdksrc", System.getProperty("jdksrc")
systemProperty "packageSources", System.getProperty("packageSources")
}.assertNormalExitValue()
}
}
Expand Down
45 changes: 39 additions & 6 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/ContextApiSpecGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ContextApiSpecGenerator {
public static void main(String[] args) throws IOException {
Expand Down Expand Up @@ -77,28 +80,58 @@ private static JavaClassFilesystemResolver getJdkSrc() {
if (jdksrc == null || "".equals(jdksrc)) {
return null;
}
return new JavaClassFilesystemResolver(PathUtils.get(jdksrc));
String packageSourcesString = System.getProperty("packageSources");
if (packageSourcesString == null || "".equals(packageSourcesString)) {
return new JavaClassFilesystemResolver(PathUtils.get(jdksrc));
}
HashMap<String, Path> packageSources = new HashMap<>();
for (String packageSourceString: packageSourcesString.split(";")) {
String[] packageSource = packageSourceString.split(":", 2);
if (packageSource.length != 2) {
throw new IllegalArgumentException(
"Bad format for packageSources. Format <package0>:<path0>;<package1>:<path1> ..."
);
}
packageSources.put(packageSource[0], PathUtils.get(packageSource[1]));
}
return new JavaClassFilesystemResolver(PathUtils.get(jdksrc), packageSources);
}

public static class JavaClassFilesystemResolver implements JavaClassResolver {
private final Path root;
private final Map<String, Path> pkgRoots;

public JavaClassFilesystemResolver(Path root) {
this.root = root;
this.pkgRoots = Collections.emptyMap();
}

public JavaClassFilesystemResolver(Path root, Map<String, Path> pkgRoots) {
this.root = root;
this.pkgRoots = pkgRoots;
}

@SuppressForbidden(reason = "resolve class file from java src directory with environment")
public InputStream openClassFile(String className) throws IOException {
// TODO(stu): handle primitives & not stdlib
if (className.contains(".") && className.startsWith("java")) {
if (className.contains(".")) {
int dollarPosition = className.indexOf("$");
if (dollarPosition >= 0) {
className = className.substring(0, dollarPosition);
}
String[] packages = className.split("\\.");
String path = String.join("/", packages);
Path classPath = root.resolve(path + ".java");
return new FileInputStream(classPath.toFile());
if (className.startsWith("java")) {
String[] packages = className.split("\\.");
String path = String.join("/", packages);
Path classPath = root.resolve(path + ".java");
return new FileInputStream(classPath.toFile());
} else {
String packageName = className.substring(0, className.lastIndexOf("."));
Path root = pkgRoots.get(packageName);
if (root != null) {
Path classPath = root.resolve(className.substring(className.lastIndexOf(".") + 1) + ".java");
return new FileInputStream(classPath.toFile());
}
}
}
return new InputStream() {
@Override
Expand Down
110 changes: 80 additions & 30 deletions modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,24 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

public class JavadocExtractor {

private final JavaClassResolver resolver;
private final Map<String, ParsedJavaClass> cache = new HashMap<>();

private static final String GPLv2 = "* This code is free software; you can redistribute it and/or modify it"+
"\n * under the terms of the GNU General Public License version 2 only, as"+
"\n * published by the Free Software Foundation.";
private static final String GPLv2 = "This code is free software; you can redistribute it and/or" +
" modify it under the terms of the GNU General Public License version 2 only, as published" +
" by the Free Software Foundation.";

private static final String ESv2 = "Copyright Elasticsearch B.V. and/or licensed to Elasticsearch" +
" B.V. under one or more contributor license agreements. Licensed under the Elastic License 2.0" +
" and the Server Side Public License, v 1; you may not use this file except in compliance with," +
" at your election, the Elastic License 2.0 or the Server Side Public License, v 1.";

private static final String[] LICENSES = new String[]{GPLv2, ESv2};

public JavadocExtractor(JavaClassResolver resolver) {
this.resolver = resolver;
Expand All @@ -58,7 +66,7 @@ public ParsedJavaClass parseClass(String className) throws IOException {
return parsed;
}
InputStream classStream = resolver.openClassFile(className);
parsed = new ParsedJavaClass(GPLv2);
parsed = new ParsedJavaClass();
if (classStream != null) {
ClassFileVisitor visitor = new ClassFileVisitor();
CompilationUnit cu = StaticJavaParser.parse(classStream);
Expand All @@ -69,32 +77,53 @@ public ParsedJavaClass parseClass(String className) throws IOException {
}

public static class ParsedJavaClass {
private static final Pattern LICENSE_CLEANUP = Pattern.compile("\\s*\n\\s*\\*");

public final Map<MethodSignature, ParsedMethod> methods;
public final Map<String, String> fields;
public final Map<List<String>, ParsedMethod> constructors;
private final String license;
private final String[] licenses;
private boolean valid = false;
private boolean validated = false;

public ParsedJavaClass(String license) {
public ParsedJavaClass(String ... licenses) {
methods = new HashMap<>();
fields = new HashMap<>();
constructors = new HashMap<>();
this.license = license;
if (licenses.length > 0) {
this.licenses = licenses;
} else {
this.licenses = LICENSES;
}
}

public void validateLicense(Optional<Comment> license) {
if (validated) {
throw new IllegalStateException("Cannot double validate the license");
}
this.valid = license.map(Comment::getContent).orElse("").contains(this.license);
String header = license.map(c -> LICENSE_CLEANUP.matcher(c.getContent()).replaceAll("").trim()).orElse("");
for (String validLicense : licenses) {
valid |= header.contains(validLicense);
}
validated = true;
}

public ParsedMethod getMethod(String name, List<String> parameterTypes) {
return methods.get(new MethodSignature(name, parameterTypes));
}

public ParsedMethod getAugmentedMethod(String methodName, String receiverType, List<String> parameterTypes) {
List<String> parameterKey = new ArrayList<>(parameterTypes.size() + 1);
parameterKey.add(receiverType);
parameterKey.addAll(parameterTypes);

ParsedMethod augmented = getMethod(methodName, parameterKey);
if (augmented == null) {
return null;
}
return augmented.asAugmented();
}

@Override
public String toString() {
return "ParsedJavaClass{" +
Expand All @@ -112,7 +141,7 @@ public void putMethod(MethodDeclaration declaration) {
declaration.getJavadoc().map(JavadocExtractor::clean).orElse(null),
declaration.getParameters()
.stream()
.map(p -> p.getName().asString())
.map(p -> stripTypeParameters(p.getName().asString()))
.collect(Collectors.toList())
)
);
Expand All @@ -134,26 +163,6 @@ public void putConstructor(ConstructorDeclaration declaration) {
);
}

private static String stripTypeParameters(String type) {
int start = 0;
int count = 0;
for (int i=0; i<type.length(); i++) {
char c = type.charAt(i);
if (c == '<') {
if (start == 0) {
start = i;
}
count++;
} else if (c == '>') {
count--;
if (count == 0) {
return type.substring(0, start);
}
}
}
return type;
}

public ParsedMethod getConstructor(List<String> parameterTypes) {
return constructors.get(parameterTypes);
}
Expand All @@ -172,6 +181,26 @@ public void putField(FieldDeclaration declaration) {
}
}

private static String stripTypeParameters(String type) {
int start = 0;
int count = 0;
for (int i=0; i<type.length(); i++) {
char c = type.charAt(i);
if (c == '<') {
if (start == 0) {
start = i;
}
count++;
} else if (c == '>') {
count--;
if (count == 0) {
return type.substring(0, start);
}
}
}
return type;
}

public static class MethodSignature {
public final String name;
public final List<String> parameterTypes;
Expand All @@ -186,7 +215,7 @@ public static MethodSignature fromDeclaration(MethodDeclaration declaration) {
declaration.getNameAsString(),
declaration.getParameters()
.stream()
.map(p -> p.getType().asString())
.map(p -> stripTypeParameters(p.getType().asString()))
.collect(Collectors.toList())
);
}
Expand Down Expand Up @@ -215,6 +244,16 @@ public ParsedMethod(ParsedJavadoc javadoc, List<String> parameterNames) {
this.parameterNames = parameterNames;
}

public ParsedMethod asAugmented() {
if (parameterNames.size() == 0) {
throw new IllegalStateException("Cannot augment without receiver: javadoc=" + javadoc);
}
return new ParsedMethod(
javadoc == null ? null : javadoc.asAugmented(parameterNames.get(0)),
new ArrayList<>(parameterNames.subList(1, parameterNames.size()))
);
}

public boolean isEmpty() {
return (javadoc == null || javadoc.isEmpty()) && parameterNames.isEmpty();
}
Expand All @@ -235,6 +274,17 @@ public ParsedJavadoc(String description) {
this.description = description;
}

public ParsedJavadoc asAugmented(String receiverName) {
if (param == null) {
return this;
}
ParsedJavadoc augmented = new ParsedJavadoc(description);
augmented.param.putAll(param);
augmented.param.remove(receiverName);
augmented.thrws = thrws;
return augmented;
}

public boolean isEmpty() {
return param.size() == 0 &&
(description == null || description.isEmpty()) &&
Expand Down
6 changes: 5 additions & 1 deletion modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,11 @@ public static List<Method> fromInfos(

JavadocExtractor.ParsedMethod parsedMethod = parsed.getMethod(name, parameterTypes);
if ((parsedMethod == null || parsedMethod.isEmpty()) && className.equals(info.getDeclaring()) == false) {
parsedMethod = extractor.parseClass(info.getDeclaring()).getMethod(name, parameterTypes);
JavadocExtractor.ParsedJavaClass parsedDeclared = extractor.parseClass(info.getDeclaring());
parsedMethod = parsedDeclared.getMethod(name, parameterTypes);
if (parsedMethod == null) {
parsedMethod = parsedDeclared.getAugmentedMethod(name, javaNamesToDisplayNames.get(className), parameterTypes);
}
}
if (parsedMethod != null) {
javadoc = parsedMethod.javadoc;
Expand Down

0 comments on commit 16a1662

Please sign in to comment.