Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve parsing of operator and function templates #732

Merged
merged 6 commits into from
Jan 1, 2024
+169 −89
Merged

Improve parsing of operator and function templates #732

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dfffed0
Improve parsing of function templates
HGuillemet Dec 24, 2023
2eb523d
Remove redundant infomap query
HGuillemet Dec 30, 2023
9623631
Replace Templates.SplitResult with plain list
HGuillemet Dec 30, 2023
514aeb9
Don't set context.virtualize for constructors.
HGuillemet Dec 30, 2023
95ff339
Instantiate templates with Info.define
HGuillemet Dec 31, 2023
7395919
Update CHANGELOG.md and fix nits
saudet Dec 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/org/bytedeco/javacpp/tools/DeclarationList.java
View file
Original file line number Diff line number Diff line change
@@ -70,7 +70,7 @@ public boolean add(Declaration decl, String fullName) {
if (infoIterator == null) {
Type type = templateMap.type = decl.type;
Declarator dcl = templateMap.declarator = decl.declarator;
for (String name : new String[] {fullName, dcl != null ? dcl.cppName : type.cppName}) {
for (String name : new String[] {fullName, dcl != null ? (dcl.type.constructor ? Parser.constructorName(dcl.cppName) : dcl.cppName) : type.cppName}) {
if (name == null) {
continue;
}
50 changes: 13 additions & 37 deletions src/main/java/org/bytedeco/javacpp/tools/InfoMap.java
View file
Original file line number Diff line number Diff line change
@@ -185,6 +185,19 @@ String normalize(String name, boolean unconst, boolean untemplate) {
if (name == null || name.length() == 0 || name.startsWith("basic/")) {
return name;
}
if (untemplate) {
// Remove template arguments in the last NS component only, and not in parameters, if any
Templates.SplitResult comps = Templates.splitNamespace(name);
int last = comps.size()-1;
String lastComp = comps.get(last);
comps.set(last, Templates.strip(lastComp));
name = comps.get(0);
for (int i = 1; i <= last; i++)
name += "::" + comps.get(i);
if (comps.parameterList != null)
name += comps.parameterList;
if (name.isEmpty()) return name;
}
boolean foundConst = false, simpleType = true;
String prefix = null;
Token[] tokens = new Tokenizer(name, null, 0).tokenize();
@@ -216,43 +229,6 @@ String normalize(String name, boolean unconst, boolean untemplate) {
for (int i = 1; i < n; i++) {
name += " " + tokens[i].value;
}
} else if (untemplate) {
int count = 0, lastColon = -1, template = -1, parameters = n;
for (int i = 0; i < n; i++) {
if (tokens[i].match('<')) {
count++;
} else if (tokens[i].match('>')) {
count--;
}
if (count == 0 && tokens[i].match("::")) {
lastColon = i;
} else if (count == 0 && tokens[i].match('(')) {
parameters = i;
break;
}
}
for (int i = lastColon + 1; i < parameters; i++) {
if (tokens[i].match('<')) {
if (count == 0) {
template = i;
}
count++;
} else if (tokens[i].match('>')) {
count--;
if (count == 0 && i + 1 != parameters) {
template = -1;
}
}
}
if (template >= 0) {
name = foundConst ? "const " : "";
for (int i = 0; i < template; i++) {
name += tokens[i];
}
for (int i = parameters; i < n; i++) {
name += tokens[i].spacing + tokens[i];
}
}
}
if (unconst && foundConst) {
name = name.substring(name.indexOf("const") + 5);
116 changes: 81 additions & 35 deletions src/main/java/org/bytedeco/javacpp/tools/Parser.java
View file
Original file line number Diff line number Diff line change
@@ -124,7 +124,17 @@ static String upcastMethodName(String javaName) {
return "as" + Character.toUpperCase(shortName.charAt(0)) + shortName.substring(1);
}

/** Returns the name of the constructor of class cppName, to be used as keys in infoMap */
/**
* Constructors have 2 kinds of fully qualified name:
* the calling name of a constructor, used when calling the constructor, e.g.:
* NS::CN(int)
* and the declaration name, used when defining the constructor outside its class or when referencing a constructor
* with using to inherit constructors of base class.
* NS::CN::CN(int)
* Declarator.cppName contains the calling name, and this method returns the declaration name.
* Keys in info map should use the declaration name, because the calling name cannot specify
* arguments in case of constructor templates, and to avoid confusion between classes and constructores info.
*/
static String constructorName(String cppName) {
String constructorName = Templates.strip(cppName);
int namespace = constructorName.lastIndexOf("::");
@@ -752,6 +762,28 @@ Type[] templateArguments(Context context) throws ParserException {
return arguments.toArray(new Type[0]);
}

/**
* Read and return the operator following an operator keyword:
* any of new, delete, + - * / % ^ & | ~ ! = < > += -= *= /= %= ^= &= |= << >> >>= <<= == != <= >= <=>(since C++20) && || ++ -- , ->* -> ( ) [ ]
* taking care of template arguments, if any.
*/
private String operator(Context context) throws ParserException {
String res = tokens.get().toString(); // Can be '('
int lenFirstToken = res.length();
String s = "";
tokens.next();
int backIndex = tokens.index;
for (Token token = tokens.get(); !token.match('(', ';', Token.EOF); token = tokens.next()) {
s += token;
}
s = Templates.strip(s);
tokens.index = backIndex;
for (Token token = tokens.get(); s.length() > res.length() - lenFirstToken; token = tokens.next()) {
res += token;
}
return res;
}

Type type(Context context) throws ParserException {
return type(context, false);
}
@@ -865,7 +897,8 @@ Type type(Context context, boolean definition) throws ParserException {
} else if (type.cppName.endsWith("::")) {
type.operator = true;
tokens.next();
break;
type.cppName += operator(context);
continue;
} else {
break;
}
@@ -957,7 +990,7 @@ Type type(Context context, boolean definition) throws ParserException {

// perform template substitution
if (context.templateMap != null) {
List<String> types = Templates.splitNamespace(type.cppName);
Templates.SplitResult types = Templates.splitNamespace(type.cppName);
String separator = "";
type.cppName = "";
List<Type> arguments = new ArrayList<>();
@@ -969,6 +1002,7 @@ Type type(Context context, boolean definition) throws ParserException {
}
separator = "::";
}
if (types.parameterList != null) type.cppName += types.parameterList;
if (arguments.size() > 0) {
type.arguments = arguments.toArray(new Type[0]);
}
@@ -1109,22 +1143,15 @@ Type type(Context context, boolean definition) throws ParserException {
}
}
if (context.cppName != null && type.javaName.length() > 0) {
String cppName = type.cppName;
String groupName = context.cppName;
String cppNameStripped = Templates.strip(cppName);
String groupNameStripped = Templates.strip(groupName);
if (cppNameStripped.length() == cppName.length() && groupNameStripped.length() != groupName.length()) {
groupName = groupNameStripped;
} else if (cppNameStripped.length() != cppName.length() && groupNameStripped.length() == groupName.length()) {
cppName = cppNameStripped;
}
String cppName = Templates.strip(type.cppName);
String groupName = Templates.strip(context.cppName);
List<String> cppNameSplit = Templates.splitNamespace(cppName);
List<String> groupNameSplit = Templates.splitNamespace(groupName);
if (cppNameSplit.size() == 1 && groupNameSplit.size() > 1)
groupName = groupNameSplit.get(groupNameSplit.size() - 1);
else if (cppNameSplit.size() > 1 && groupNameSplit.size() == 1)
cppName = cppNameSplit.get(cppNameSplit.size() - 1);
if (cppName.equals(groupName) || groupName.startsWith(cppName + "<")) {
if (cppName.equals(groupName)) {
type.constructor = !type.destructor && !type.operator
&& type.indirections == 0 && !type.reference && tokens.get().match('(', ':');
}
@@ -1347,11 +1374,8 @@ Declarator declarator(Context context, String defaultName, int infoNumber, boole
} else if (token.match(Token.OPERATOR)) {
dcl.operator = true;
if (!tokens.get(1).match(Token.IDENTIFIER) || tokens.get(1).match(Token.NEW, Token.DELETE)) {
// assume we can have any symbols until the first open parenthesis
dcl.cppName += "operator " + tokens.next();
for (token = tokens.next(); !token.match(Token.EOF, '('); token = tokens.next()) {
dcl.cppName += token;
}
tokens.next();
dcl.cppName += "operator " + operator(context);
break;
}
} else if (token.match('<')) {
@@ -2130,14 +2154,15 @@ Parameters parameters(Context context, int infoNumber, boolean useDefaults) thro
// perform template substitution
String cppName = token.value;
if (context.templateMap != null) {
List<String> types = Templates.splitNamespace(cppName);
Templates.SplitResult types = Templates.splitNamespace(cppName);
String separator = "";
cppName = "";
for (String t : types) {
Type t2 = context.templateMap.get(t);
cppName += separator + (t2 != null ? t2.cppName : t);
separator = "::";
}
if (types.parameterList != null) cppName += types.parameterList;
}

// try to qualify all the identifiers
@@ -2343,14 +2368,16 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
}

boolean isQualified = Templates.splitNamespace(dcl.cppName).size() > 1;
if (context.namespace != null && !isQualified) {
if (context.namespace != null && !isQualified && !(type.constructor || type.destructor)) {
dcl.cppName = context.namespace + "::" + dcl.cppName;
}
Info info = null, fullInfo = null;
String fullname = dcl.cppName, fullname2 = dcl.cppName;
String templateArgs = declList.templateMap != null ? declList.templateMap.toString() : "";
String fullname = dcl.cppName + templateArgs;
String param1 = "", param2 = "";
if (dcl.parameters != null) {
fullname += "(";
fullname2 += "(";
param1 = "(";
param2 = "(";
String separator = "";
for (Declarator d : dcl.parameters.declarators) {
if (d != null) {
@@ -2376,25 +2403,45 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
if (d.type.constPointer && !s.endsWith(" const")) {
s = s + " const";
}
fullname += separator + s;
fullname2 += separator + s2;
param1 += separator + s;
param2 += separator + s2;
separator = ", ";
}
}
info = fullInfo = infoMap.getFirst(fullname += ")", false);
param1 += ")";
param2 += ")";
fullname += param1;
info = fullInfo = infoMap.getFirst(fullname, false);
if (info == null) {
info = infoMap.getFirst(fullname2 += ")", false);
info = infoMap.getFirst(dcl.cppName + templateArgs + param2, false);
if (info == null && !templateArgs.isEmpty()) {
info = infoMap.getFirst(dcl.cppName + param1, false);
if (info == null) {
info = infoMap.getFirst(dcl.cppName + param2, false);
}
}
}
}
if (info == null) {
if (type.constructor) {
// get Info explicitly associated with all constructors
List<String> cppNameSplit = Templates.splitNamespace(dcl.cppName);
String name = Templates.strip(cppNameSplit.get(cppNameSplit.size() - 1));
info = fullInfo = infoMap.getFirst(dcl.cppName + "::" + name);
String name = constructorName(dcl.cppName);
fullname = name + templateArgs + param1;
info = fullInfo = infoMap.getFirst(fullname);
if (info == null) {
info = fullInfo = infoMap.getFirst(name + templateArgs + param2);
if (info == null) {
info = fullInfo = infoMap.getFirst(name + templateArgs);
}
}
}
// For constructor, we'd better not make this lookup, because of confusion
// with the class info. Kept for now for backwards compatibility.
if (info == null) {
info = infoMap.getFirst(dcl.cppName);
info = infoMap.getFirst(dcl.cppName + templateArgs);
if (info == null && !templateArgs.isEmpty()) {
info = infoMap.getFirst(dcl.cppName);
}
}
if (!type.constructor && !type.destructor && !type.operator && (context.templateMap == null || context.templateMap.full())) {
infoMap.put(info != null ? new Info(info).cppNames(fullname).javaNames(null) : new Info(fullname));
@@ -2526,8 +2573,7 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
}

// use Java names that we may get here but that declarator() did not catch
String parameters = fullname.substring(dcl.cppName.length());
for (String name : context.qualify(dcl.cppName, parameters)) {
for (String name : context.qualify(dcl.cppName, param1)) {
if ((infoMap.getFirst(name, false)) != null) {
dcl.cppName = name;
break;
@@ -2539,8 +2585,8 @@ boolean function(Context context, DeclarationList declList) throws ParserExcepti
if (context.namespace != null && localName2.startsWith(context.namespace + "::")) {
localName2 = dcl.cppName.substring(context.namespace.length() + 2);
}
if (localName2.endsWith(parameters)) {
localName2 = localName2.substring(0, localName2.length() - parameters.length());
if (localName2.endsWith(param1)) {
localName2 = localName2.substring(0, localName2.length() - param1.length());
}
if (fullInfo != null && fullInfo.javaNames != null && fullInfo.javaNames.length > 0) {
dcl.javaName = fullInfo.javaNames[0];
16 changes: 16 additions & 0 deletions src/main/java/org/bytedeco/javacpp/tools/TemplateMap.java
View file
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
package org.bytedeco.javacpp.tools;

import java.util.LinkedHashMap;
import java.util.Map;

/**
*
@@ -67,4 +68,19 @@ Type get(String key) {
return value;
}
}

@Override
public String toString() {
String res = "<";
for (Map.Entry<String, Type> e: entrySet()) {
if (res.length() > 1) res += ",";
Type t = e.getValue();
if (t == null)
res += e.getKey();
else
res += t.cppName;
}
if (res.charAt(res.length()-1) == '>') res += " ";
return res + ">";
}
}
32 changes: 27 additions & 5 deletions src/main/java/org/bytedeco/javacpp/tools/Templates.java
View file
Original file line number Diff line number Diff line change
@@ -47,8 +47,12 @@ static boolean notExists(String s) {
return strip(s).length() == s.length();
}

/** Split s at ::, but taking care of qualified template arguments */
static List<String> splitNamespace(String s) {
static class SplitResult extends ArrayList<String> {
String parameterList;
}

/** Split s at ::, but taking care of qualified template arguments and qualified function parameters, if any. */
static SplitResult splitNamespace(String s) {
String sTemplatesMasked = s;
for (;;) {
Matcher m = templatePattern.matcher(sTemplatesMasked);
@@ -60,18 +64,36 @@ static List<String> splitNamespace(String s) {
break;
}
}
ArrayList<String> comps = new ArrayList<>();
SplitResult comps = new SplitResult();
int pIndex = sTemplatesMasked.lastIndexOf(')'); // last because of function pointer types like void(*)()
if (pIndex > 0) {
// Pointer list may contain function pointer types with parentheses
int count = 1;
for (pIndex--; pIndex >= 0; pIndex--) {
char c = sTemplatesMasked.charAt(pIndex);
if (c == ')') count++;
else if (c == '(') {
count--;
if (count == 0) break;
}
}
}
int start = 0;
for (;;) {
int i = sTemplatesMasked.indexOf("::", start);
if (i >= 0) {
if (i >= 0 && (i < pIndex || pIndex == -1)) {
comps.add(s.substring(start, i));
start = i + 2;
} else {
break;
}
}
comps.add(s.substring(start));
if (pIndex >= 0) {
comps.add(s.substring(start, pIndex));
comps.parameterList = s.substring(pIndex);
} else {
comps.add(s.substring(start));
}
return comps;
}
}