2
* Licensed to the Apache Software Foundation (ASF) under one or more
3
* contributor license agreements. See the NOTICE file distributed with
4
* this work for additional information regarding copyright ownership.
5
* The ASF licenses this file to You under the Apache License, Version 2.0
6
* (the "License"); you may not use this file except in compliance with
7
* the License. You may obtain a copy of the License at
9
* http://www.apache.org/licenses/LICENSE-2.0
11
* Unless required by applicable law or agreed to in writing, software
12
* distributed under the License is distributed on an "AS IS" BASIS,
13
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
* See the License for the specific language governing permissions and
15
* limitations under the License.
18
package org.apache.solr.search.function;
20
import org.apache.lucene.search.FieldCache;
21
import org.apache.solr.SolrTestCaseJ4;
22
import org.apache.solr.common.params.SolrParams;
23
import org.apache.solr.common.util.NamedList;
24
import org.junit.BeforeClass;
25
import org.junit.Test;
26
import java.io.FileOutputStream;
27
import java.io.OutputStreamWriter;
28
import java.io.Writer;
29
import java.util.ArrayList;
30
import java.util.Arrays;
31
import java.util.List;
32
import java.util.Random;
35
* Tests some basic functionality of Solr while demonstrating good
36
* Best Practices for using AbstractSolrTestCase
38
public class TestFunctionQuery extends SolrTestCaseJ4 {
40
public static void beforeClass() throws Exception {
41
initCore("solrconfig-functionquery.xml","schema11.xml");
45
public void setUp() throws Exception {
50
String base = "external_foo_extf";
51
static long start = System.currentTimeMillis();
52
void makeExternalFile(String field, String contents, String charset) {
53
String dir = h.getCore().getDataDir();
54
String filename = dir + "/external_" + field + "." + (start++);
56
Writer out = new OutputStreamWriter(new FileOutputStream(filename), charset);
59
} catch (Exception e) {
60
throw new RuntimeException(e);
65
void createIndex(String field, float... values) {
66
// lrf.args.put("version","2.0");
67
for (float val : values) {
68
String s = Float.toString(val);
69
if (field!=null) assertU(adoc("id", s, field, s));
70
else assertU(adoc("id", s));
71
// System.out.println("added doc for " + val);
73
assertU(optimize()); // squeeze out any possible deleted docs
76
// replace \0 with the field name and create a parseable string
77
public String func(String field, String template) {
78
StringBuilder sb = new StringBuilder("_val_:\"");
79
for (char ch : template.toCharArray()) {
84
if (ch=='"') sb.append('\\');
91
void singleTest(String field, String funcTemplate, List<String> args, float... results) {
92
String parseableQuery = func(field, funcTemplate);
94
List<String> nargs = new ArrayList<String>(Arrays.asList("q", parseableQuery
100
for (String arg : args) {
101
nargs.add(arg.replace("\0",field));
105
List<String> tests = new ArrayList<String>();
107
// Construct xpaths like the following:
108
// "//doc[./float[@name='foo_pf']='10.0' and ./float[@name='score']='10.0']"
110
for (int i=0; i<results.length; i+=2) {
111
String xpath = "//doc[./float[@name='" + "id" + "']='"
112
+ results[i] + "' and ./float[@name='score']='"
113
+ results[i+1] + "']";
117
assertQ(req(nargs.toArray(new String[]{}))
118
, tests.toArray(new String[]{})
122
void singleTest(String field, String funcTemplate, float... results) {
123
singleTest(field, funcTemplate, null, results);
126
void doTest(String field) {
127
// lrf.args.put("version","2.0");
128
float[] vals = new float[] {
131
createIndex(field,vals);
132
createIndex(null, 88); // id with no value
134
// test identity (straight field value)
135
singleTest(field, "\0", 10,10);
137
// test constant score
138
singleTest(field,"1.414213", 10, 1.414213f);
139
singleTest(field,"-1.414213", 10, -1.414213f);
141
singleTest(field,"sum(\0,1)", 10, 11);
142
singleTest(field,"sum(\0,\0)", 10, 20);
143
singleTest(field,"sum(\0,\0,5)", 10, 25);
145
singleTest(field,"sub(\0,1)", 10, 9);
147
singleTest(field,"product(\0,1)", 10, 10);
148
singleTest(field,"product(\0,-2,-4)", 10, 80);
150
singleTest(field,"log(\0)",10,1, 100,2);
151
singleTest(field,"sqrt(\0)",100,10, 25,5, 0,0);
152
singleTest(field,"abs(\0)",10,10, -4,4);
153
singleTest(field,"pow(\0,\0)",0,1, 5,3125);
154
singleTest(field,"pow(\0,0.5)",100,10, 25,5, 0,0);
155
singleTest(field,"div(1,\0)",-4,-.25f, 10,.1f, 100,.01f);
156
singleTest(field,"div(1,1)",-4,1, 10,1);
158
singleTest(field,"sqrt(abs(\0))",-4,2);
159
singleTest(field,"sqrt(sum(29,\0))",-4,5);
161
singleTest(field,"map(\0,0,0,500)",10,10, -4,-4, 0,500);
162
singleTest(field,"map(\0,-4,5,500)",100,100, -4,500, 0,500, 5,500, 10,10, 25,25);
164
singleTest(field,"scale(\0,-1,1)",-4,-1, 100,1, 0,-0.9230769f);
165
singleTest(field,"scale(\0,-10,1000)",-4,-10, 100,1000, 0,28.846153f);
167
// test that infinity doesn't mess up scale function
168
singleTest(field,"scale(log(\0),-1000,1000)",100,1000);
170
// test use of an ValueSourceParser plugin: nvl function
171
singleTest(field,"nvl(\0,1)", 0, 1, 100, 100);
173
// compose the ValueSourceParser plugin function with another function
174
singleTest(field, "nvl(sum(0,\0),1)", 0, 1, 100, 100);
176
// test simple embedded query
177
singleTest(field,"query({!func v=\0})", 10, 10, 88, 0);
178
// test default value for embedded query
179
singleTest(field,"query({!lucene v='\0:[* TO *]'},8)", 88, 8);
180
singleTest(field,"sum(query({!func v=\0},7.1),query({!func v=\0}))", 10, 20, 100, 200);
181
// test with sub-queries specified by other request args
182
singleTest(field,"query({!func v=$vv})", Arrays.asList("vv","\0"), 10, 10, 88, 0);
183
singleTest(field,"query($vv)",Arrays.asList("vv","{!func}\0"), 10, 10, 88, 0);
184
singleTest(field,"sum(query($v1,5),query($v1,7))",
185
Arrays.asList("v1","\0:[* TO *]"), 88,12
188
purgeFieldCache(FieldCache.DEFAULT); // avoid FC insanity
192
public void testFunctions() {
193
doTest("foo_pf"); // a plain float field
194
doTest("foo_f"); // a sortable float field
195
doTest("foo_tf"); // a trie float field
199
public void testExternalField() throws Exception {
200
String field = "foo_extf";
202
float[] ids = {100,-4,0,10,25,5,77,23,55,-78,-45,-24,63,78,94,22,34,54321,261,-627};
204
createIndex(null,ids);
206
// Unsorted field, largest first
207
makeExternalFile(field, "54321=543210\n0=-999\n25=250","UTF-8");
208
// test identity (straight field value)
209
singleTest(field, "\0", 54321, 543210, 0,-999, 25,250, 100, 1);
210
Object orig = FileFloatSource.onlyForTesting;
211
singleTest(field, "log(\0)");
212
// make sure the values were cached
213
assertTrue(orig == FileFloatSource.onlyForTesting);
214
singleTest(field, "sqrt(\0)");
215
assertTrue(orig == FileFloatSource.onlyForTesting);
217
makeExternalFile(field, "0=1","UTF-8");
218
assertU(h.query("/reloadCache",lrf.makeRequest("","")));
219
singleTest(field, "sqrt(\0)");
220
assertTrue(orig != FileFloatSource.onlyForTesting);
224
for (int i=0; i<10; i++) { // do more iterations for a thorough test
225
int len = r.nextInt(ids.length+1);
226
boolean sorted = r.nextBoolean();
228
for (int j=0; j<ids.length; j++) {
229
int other=r.nextInt(ids.length);
236
// sort only the first elements
237
Arrays.sort(ids,0,len);
240
// make random values
241
float[] vals = new float[len];
242
for (int j=0; j<len; j++) {
243
vals[j] = r.nextInt(200)-100;
246
// make and write the external file
247
StringBuilder sb = new StringBuilder();
248
for (int j=0; j<len; j++) {
249
sb.append("" + ids[j] + "=" + vals[j]+"\n");
251
makeExternalFile(field, sb.toString(),"UTF-8");
254
assertU(h.query("/reloadCache",lrf.makeRequest("","")));
257
float[] answers = new float[ids.length*2];
258
for (int j=0; j<len; j++) {
259
answers[j*2] = ids[j];
260
answers[j*2+1] = vals[j];
262
for (int j=len; j<ids.length; j++) {
263
answers[j*2] = ids[j];
264
answers[j*2+1] = 1; // the default values
267
singleTest(field, "\0", answers);
268
// System.out.println("Done test "+i);
271
purgeFieldCache(FieldCache.DEFAULT); // avoid FC insanity
275
public void testExternalFileFieldStringKeys() throws Exception {
276
final String extField = "foo_extfs";
277
final String keyField = "sfile_s";
278
assertU(adoc("id", "991", keyField, "AAA=AAA"));
279
assertU(adoc("id", "992", keyField, "BBB"));
280
assertU(adoc("id", "993", keyField, "CCC=CCC"));
282
makeExternalFile(extField, "AAA=AAA=543210\nBBB=-8\nCCC=CCC=250","UTF-8");
283
singleTest(extField,"\0",991,543210,992,-8,993,250);
287
public void testGeneral() throws Exception {
290
assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z"));
291
assertU(adoc("id","2"));
292
assertU(commit()); // create more than one segment
293
assertU(adoc("id","3"));
294
assertU(adoc("id","4"));
295
assertU(commit()); // create more than one segment
296
assertU(adoc("id","5"));
297
assertU(adoc("id","6"));
300
// test that ord and rord are working on a global index basis, not just
301
// at the segment level (since Lucene 2.9 has switched to per-segment searching)
302
assertQ(req("fl","*,score","q", "{!func}ord(id)", "fq","id:6"), "//float[@name='score']='6.0'");
303
assertQ(req("fl","*,score","q", "{!func}top(ord(id))", "fq","id:6"), "//float[@name='score']='6.0'");
304
assertQ(req("fl","*,score","q", "{!func}rord(id)", "fq","id:1"),"//float[@name='score']='6.0'");
305
assertQ(req("fl","*,score","q", "{!func}top(rord(id))", "fq","id:1"),"//float[@name='score']='6.0'");
308
// test that we can subtract dates to millisecond precision
309
assertQ(req("fl","*,score","q", "{!func}ms(a_tdt,b_tdt)", "fq","id:1"), "//float[@name='score']='-1.0'");
310
assertQ(req("fl","*,score","q", "{!func}ms(b_tdt,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
311
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,2009-08-31T12:10:10.124Z)", "fq","id:1"), "//float[@name='score']='1.0'");
312
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.124Z,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
313
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,b_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
315
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z/SECOND,2009-08-31T12:10:10.124Z/SECOND)", "fq","id:1"), "//float[@name='score']='0.0'");
317
for (int i=100; i<112; i++) {
318
assertU(adoc("id",""+i, "text","batman"));
321
assertU(adoc("id","120", "text","batman superman")); // in a smaller segment
322
assertU(adoc("id","121", "text","superman"));
325
// superman has a higher df (thus lower idf) in one segment, but reversed in the complete index
326
String q ="{!func}query($qq)";
328
assertQ(req("fl","*,score","q", q, "qq","text:batman", "fq",fq), "//float[@name='score']<'1.0'");
329
assertQ(req("fl","*,score","q", q, "qq","text:superman", "fq",fq), "//float[@name='score']>'1.0'");
331
// test weighting through a function range query
332
assertQ(req("fl","*,score", "fq",fq, "q", "{!frange l=1 u=10}query($qq)", "qq","text:superman"), "//*[@numFound='1']");
334
// test weighting through a complex function
335
q ="{!func}sub(div(sum(0.0,product(1,query($qq))),1),0)";
336
assertQ(req("fl","*,score","q", q, "qq","text:batman", "fq",fq), "//float[@name='score']<'1.0'");
337
assertQ(req("fl","*,score","q", q, "qq","text:superman", "fq",fq), "//float[@name='score']>'1.0'");
340
// test full param dereferencing
341
assertQ(req("fl","*,score","q", "{!func}add($v1,$v2)", "v1","add($v3,$v4)", "v2","1", "v3","2", "v4","5"
342
, "fq","id:1"), "//float[@name='score']='8.0'");
344
// test ability to parse multiple values
345
assertQ(req("fl","*,score","q", "{!func}dist(2,vector(1,1),$pt)", "pt","3,1"
346
, "fq","id:1"), "//float[@name='score']='2.0'");
348
// test that extra stuff after a function causes an error
350
assertQ(req("fl","*,score","q", "{!func}10 wow dude ignore_exception"));
352
} catch (Exception e) {
356
// test that sorting by function weights correctly. superman should sort higher than batman due to idf of the whole index
358
assertQ(req("q", "*:*", "fq","id:120 OR id:121", "sort","{!func v=$sortfunc} desc", "sortfunc","query($qq)", "qq","text:(batman OR superman)")
359
,"*//doc[1]/float[.='120.0']"
360
,"*//doc[2]/float[.='121.0']"
364
purgeFieldCache(FieldCache.DEFAULT); // avoid FC insanity
368
public void testSortByFunc() throws Exception {
369
assertU(adoc("id", "1", "const_s", "xx",
370
"x_i", "100", "1_s", "a",
371
"x:x_i", "100", "1-1_s", "a"));
372
assertU(adoc("id", "2", "const_s", "xx",
373
"x_i", "300", "1_s", "c",
374
"x:x_i", "300", "1-1_s", "c"));
375
assertU(adoc("id", "3", "const_s", "xx",
376
"x_i", "200", "1_s", "b",
377
"x:x_i", "200", "1-1_s", "b"));
380
String desc = "/response/docs==[{'x_i':300},{'x_i':200},{'x_i':100}]";
381
String asc = "/response/docs==[{'x_i':100},{'x_i':200},{'x_i':300}]";
383
String threeonetwo = "/response/docs==[{'x_i':200},{'x_i':100},{'x_i':300}]";
385
String q = "id:[1 TO 3]";
386
assertJQ(req("q",q, "fl","x_i", "sort","add(x_i,x_i) desc")
390
// param sub of entire function
391
assertJQ(req("q",q, "fl","x_i", "sort", "const_s asc, $x asc", "x","add(x_i,x_i)")
395
// multiple functions
396
assertJQ(req("q",q, "fl","x_i", "sort", "$x asc, const_s asc, $y desc", "x", "5", "y","add(x_i,x_i)")
400
// multiple functions inline
401
assertJQ(req("q",q, "fl","x_i", "sort", "add( 10 , 10 ) asc, const_s asc, add(x_i , $const) desc", "const","50")
405
// test function w/ local params + func inline
406
assertJQ(req("q",q, "fl","x_i",
407
"sort", "const_s asc, {!key=foo}add(x_i,x_i) desc")
410
assertJQ(req("q",q, "fl","x_i",
411
"sort", "{!key=foo}add(x_i,x_i) desc, const_s asc")
415
// test multiple functions w/ local params + func inline
416
assertJQ(req("q",q, "fl","x_i", "sort", "{!key=bar}add(10,20) asc, const_s asc, {!key=foo}add(x_i,x_i) desc")
420
// test multiple functions w/ local param value not inlined
421
assertJQ(req("q",q, "fl","x_i", "sort", "{!key=bar v=$s1} asc, {!key=foo v=$s2} desc", "s1","add(3,4)", "s2","add(x_i,5)")
425
// no space between inlined localparams and sort order
426
assertJQ(req("q",q, "fl","x_i", "sort", "{!key=bar v=$s1}asc,const_s asc,{!key=foo v=$s2}desc", "s1","add(3,4)", "s2","add(x_i,5)")
430
// field name that isn't a legal java Identifier
431
// and starts with a number to trick function parser
432
assertJQ(req("q",q, "fl","x_i", "sort", "1_s asc")
435
assertJQ(req("q",q, "fl","x_i", "sort", "x:x_i desc")
438
assertJQ(req("q",q, "fl","x_i", "sort", "1-1_s asc")
442
// really ugly field name that isn't a java Id, and can't be
443
// parsed as a func, but sorted fine in Solr 1.4
444
assertJQ(req("q",q, "fl","x_i",
445
"sort", "[]_s asc, {!key=foo}add(x_i,x_i) desc")
448
// use localparms to sort by a lucene query, then a function
449
assertJQ(req("q",q, "fl","x_i",
450
"sort", "{!lucene v='id:3'}desc, {!key=foo}add(x_i,x_i) asc")
458
public void testDegreeRads() throws Exception {
459
assertU(adoc("id", "1", "x_td", "0", "y_td", "0"));
460
assertU(adoc("id", "2", "x_td", "90", "y_td", String.valueOf(Math.PI / 2)));
461
assertU(adoc("id", "3", "x_td", "45", "y_td", String.valueOf(Math.PI / 4)));
465
assertQ(req("fl", "*,score", "q", "{!func}rad(x_td)", "fq", "id:1"), "//float[@name='score']='0.0'");
466
assertQ(req("fl", "*,score", "q", "{!func}rad(x_td)", "fq", "id:2"), "//float[@name='score']='" + (float) (Math.PI / 2) + "'");
467
assertQ(req("fl", "*,score", "q", "{!func}rad(x_td)", "fq", "id:3"), "//float[@name='score']='" + (float) (Math.PI / 4) + "'");
469
assertQ(req("fl", "*,score", "q", "{!func}deg(y_td)", "fq", "id:1"), "//float[@name='score']='0.0'");
470
assertQ(req("fl", "*,score", "q", "{!func}deg(y_td)", "fq", "id:2"), "//float[@name='score']='90.0'");
471
assertQ(req("fl", "*,score", "q", "{!func}deg(y_td)", "fq", "id:3"), "//float[@name='score']='45.0'");
475
public void testStrDistance() throws Exception {
476
assertU(adoc("id", "1", "x_s", "foil"));
478
assertQ(req("fl", "*,score", "q", "{!func}strdist(x_s, 'foit', edit)", "fq", "id:1"), "//float[@name='score']='0.75'");
479
assertQ(req("fl", "*,score", "q", "{!func}strdist(x_s, 'foit', jw)", "fq", "id:1"), "//float[@name='score']='0.8833333'");
480
assertQ(req("fl", "*,score", "q", "{!func}strdist(x_s, 'foit', ngram, 2)", "fq", "id:1"), "//float[@name='score']='0.875'");
483
public void dofunc(String func, double val) throws Exception {
484
// String sval = Double.toString(val);
485
String sval = Float.toString((float)val);
487
assertQ(req("fl", "*,score", "defType","func", "fq","id:1", "q",func),
488
"//float[@name='score']='" + sval + "'");
492
public void testFuncs() throws Exception {
493
assertU(adoc("id", "1", "foo_d", "9"));
497
dofunc("e()", Math.E);
498
dofunc("pi()", Math.PI);
499
dofunc("add(2,3)", 2+3);
500
dofunc("mul(2,3)", 2*3);
501
dofunc("rad(45)", Math.toRadians(45));
502
dofunc("deg(.5)", Math.toDegrees(.5));
503
dofunc("sqrt(9)", Math.sqrt(9));
504
dofunc("cbrt(8)", Math.cbrt(8));
505
dofunc("max(0,1)", Math.max(0,1));
506
dofunc("max(10,3,8,7,5,4)", Math.max(Math.max(Math.max(Math.max(Math.max(10,3),8),7),5),4));
507
dofunc("min(0,1)", Math.min(0,1));
508
dofunc("min(10,3,8,7,5,4)", Math.min(Math.min(Math.min(Math.min(Math.min(10,3),8),7),5),4));
509
dofunc("log(100)", Math.log10(100));
510
dofunc("ln(3)", Math.log(3));
511
dofunc("exp(1)", Math.exp(1));
512
dofunc("sin(.5)", Math.sin(.5));
513
dofunc("cos(.5)", Math.cos(.5));
514
dofunc("tan(.5)", Math.tan(.5));
515
dofunc("asin(.5)", Math.asin(.5));
516
dofunc("acos(.5)", Math.acos(.5));
517
dofunc("atan(.5)", Math.atan(.5));
518
dofunc("sinh(.5)", Math.sinh(.5));
519
dofunc("cosh(.5)", Math.cosh(.5));
520
dofunc("tanh(.5)", Math.tanh(.5));
521
dofunc("ceil(2.3)", Math.ceil(2.3));
522
dofunc("floor(2.3)", Math.floor(2.3));
523
dofunc("rint(2.3)", Math.rint(2.3));
524
dofunc("pow(2,0.5)", Math.pow(2,0.5));
525
dofunc("hypot(3,4)", Math.hypot(3,4));
526
dofunc("atan2(.25,.5)", Math.atan2(.25,.5));