HBase – Hadoop Database，是一个高性、高性能、面向列、可伸缩的，利用HBase技术可在廉价PC Server上搭建起大规模化集群。

Lucene内置很多的分词器工具包，几乎涵盖了全球所有的国家和地区，最近散仙，在搞多语言分词的一个处理，主要国家有西班牙，葡萄牙，德语，法语，意大利，其实这些语系都与英语非常类似，都是以空格为分割的语种。

List
    
      list=new ArrayList
     
      ();		list.add("player");//这里面的词，不会被做词干抽取，词形还原		CharArraySet ar=new CharArraySet(Version.LUCENE_43,list , true);		//分词器的第二个参数是禁用词参数，第三个参数是排除不做词形转换，或单复数的词		GermanAnalyzer sa=new GermanAnalyzer(Version.LUCENE_43,null,ar);
     
    

protected TokenStreamComponents createComponents(String fieldName,      Reader reader) {	  //标准分词器过滤    final Tokenizer source = new StandardTokenizer(matchVersion, reader);    TokenStream result = new StandardFilter(matchVersion, source);	//转小写过滤    result = new LowerCaseFilter(matchVersion, result);	//禁用词过滤    result = new StopFilter( matchVersion, result, stopwords);	//排除词过滤    result = new SetKeywordMarkerFilter(result, exclusionSet);    if (matchVersion.onOrAfter(Version.LUCENE_36)) {	//在lucene3.6以后的版本，采用如下filter过滤	  //规格化，将德语中的特殊字符，映射成英语      result = new GermanNormalizationFilter(result);	  //stem词干抽取，词性还原      result = new GermanLightStemFilter(result);    } else if (matchVersion.onOrAfter(Version.LUCENE_31)) {	//在lucene3.1至3.6的版本中，采用SnowballFilter处理      result = new SnowballFilter(result, new German2Stemmer());    } else {	//在lucene3.1之前的采用兼容的GermanStemFilter处理      result = new GermanStemFilter(result);    }    return new TokenStreamComponents(source, result);  }

package org.apache.lucene.analysis.de;/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements.  See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */import java.io.IOException;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.util.StemmerUtil;/** * Normalizes German characters according to the heuristics * of the  * German2 snowball algorithm. * It allows for the fact that ä, ö and ü are sometimes written as ae, oe and ue. *  * [list] *   
 'ß' is replaced by 'ss' *   
 'ä', 'ö', 'ü' are replaced by 'a', 'o', 'u', respectively. *   
 'ae' and 'oe' are replaced by 'a', and 'o', respectively. *   
 'ue' is replaced by 'u', when not following a vowel or q. * [/list] * 
 * This is useful if you want this normalization without using * the German2 stemmer, or perhaps no stemming at all. *上面的解释说得很清楚，主要是对德文的一些特殊字母，转换成对应的英文处理 * */ public final class GermanNormalizationFilter extends TokenFilter {  // FSM with 3 states:  private static final int N = 0; /* ordinary state */  private static final int V = 1; /* stops 'u' from entering umlaut state */  private static final int U = 2; /* umlaut state, allows e-deletion */  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);    public GermanNormalizationFilter(TokenStream input) {    super(input);  }  @Override  public boolean incrementToken() throws IOException {    if (input.incrementToken()) {      int state = N;      char buffer[] = termAtt.buffer();      int length = termAtt.length();      for (int i = 0; i < length; i++) {        final char c = buffer[i];        switch(c) {          case 'a':          case 'o':            state = U;            break;          case 'u':            state = (state == N) ? U : V;            break;          case 'e':            if (state == U)              length = StemmerUtil.delete(buffer, i--, length);            state = V;            break;          case 'i':          case 'q':          case 'y':            state = V;            break;          case 'ä':            buffer[i] = 'a';            state = V;            break;          case 'ö':            buffer[i] = 'o';            state = V;            break;          case 'ü':             buffer[i] = 'u';            state = V;            break;          case 'ß':            buffer[i++] = 's';            buffer = termAtt.resizeBuffer(1+length);            if (i < length)              System.arraycopy(buffer, i, buffer, i+1, (length-i));            buffer[i] = 's';            length++;            state = N;            break;          default:            state = N;        }      }      termAtt.setLength(length);      return true;    } else {      return false;    }  }}

package org.apache.lucene.analysis.de;/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements.  See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */import java.io.IOException;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;/** * A {@link TokenFilter} that applies {@link GermanLightStemmer} to stem German * words. * 
 * To prevent terms from being stemmed use an instance of * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets * the {@link KeywordAttribute} before this {@link TokenStream}. *  * * *这个类，主要做Stemmer（词干提取），而我们主要关注 *GermanLightStemmer这个类的作用 * * */public final class GermanLightStemFilter extends TokenFilter {  private final GermanLightStemmer stemmer = new GermanLightStemmer();  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);  public GermanLightStemFilter(TokenStream input) {    super(input);  }    @Override  public boolean incrementToken() throws IOException {    if (input.incrementToken()) {      if (!keywordAttr.isKeyword()) {        final int newlen = stemmer.stem(termAtt.buffer(), termAtt.length());        termAtt.setLength(newlen);      }      return true;    } else {      return false;    }  }}

package org.apache.lucene.analysis.de;/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements.  See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License.  You may obtain a copy of the License at * *     http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. *//*  * This algorithm is updated based on code located at: * http://members.unine.ch/jacques.savoy/clef/ *  * Full copyright for that code follows: *//* * Copyright (c) 2005, Jacques Savoy * All rights reserved. * * Redistribution and use in source and binary forms, with or without  * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright notice, this  * list of conditions and the following disclaimer. Redistributions in binary  * form must reproduce the above copyright notice, this list of conditions and * the following disclaimer in the documentation and/or other materials  * provided with the distribution. Neither the name of the author nor the names  * of its contributors may be used to endorse or promote products derived from  * this software without specific prior written permission. *  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. *//** * Light Stemmer for German. * 
 * This stemmer implements the "UniNE" algorithm in: * Light Stemming Approaches for the French, Portuguese, German and Hungarian Languages * Jacques Savoy */public class GermanLightStemmer {    //处理特殊字符映射  public int stem(char s[], int len) {       for (int i = 0; i < len; i++)      switch(s[i]) {        case 'ä':        case 'à':        case 'á':        case 'â': s[i] = 'a'; break;        case 'ö':        case 'ò':        case 'ó':        case 'ô': s[i] = 'o'; break;        case 'ï':        case 'ì':        case 'í':        case 'î': s[i] = 'i'; break;        case 'ü':         case 'ù':         case 'ú':        case 'û': s[i] = 'u'; break;      }        len = step1(s, len);    return step2(s, len);  }      private boolean stEnding(char ch) {    switch(ch) {      case 'b':      case 'd':      case 'f':      case 'g':      case 'h':      case 'k':      case 'l':      case 'm':      case 'n':      case 't': return true;      default: return false;    }  }  //处理基于以下规则的词干抽取和缩减  private int step1(char s[], int len) {    if (len > 5 && s[len-3] == 'e' && s[len-2] == 'r' && s[len-1] == 'n')      return len - 3;        if (len > 4 && s[len-2] == 'e')      switch(s[len-1]) {        case 'm':        case 'n':        case 'r':        case 's': return len - 2;      }        if (len > 3 && s[len-1] == 'e')      return len - 1;        if (len > 3 && s[len-1] == 's' && stEnding(s[len-2]))      return len - 1;        return len;  }  //处理基于以下规则est，er，en等的词干抽取和缩减  private int step2(char s[], int len) {    if (len > 5 && s[len-3] == 'e' && s[len-2] == 's' && s[len-1] == 't')      return len - 3;        if (len > 4 && s[len-2] == 'e' && (s[len-1] == 'r' || s[len-1] == 'n'))      return len - 2;        if (len > 4 && s[len-2] == 's' && s[len-1] == 't' && stEnding(s[len-3]))      return len - 2;        return len;  }}

搜索技术交流群：324714439大数据hadoop交流群：3769321600，将一些德语特殊字符，替换成对应的英文表示1，将所有词干元音还原 a ，o，i，uste（2）（按先后顺序，符合以下任意一项，就完成一次校验（return））2，单词长度大于5的词，以ern结尾的，直接去掉3，单词长度大于4的词，以em，en，es，er结尾的，直接去掉4，单词长度大于3的词，以e结尾的直接去掉5，单词长度大于3的词，以bs，ds，fs，gs，hs，ks，ls，ms，ns，ts结尾的，直接去掉sstep（3）（按先后顺序，符合以下任意一项，就完成一次校验（return））6，单词长度大于5的词，以est结尾的，直接去掉7，单词长度大于4的词，以er或en结尾的直接去掉8，单词长度大于4的词，bst，dst，fst，gst，hst，kst，lst，mst，nst，tst，直接去掉后两位字母st