.NET下文本相似度算法余弦定理和SimHash浅析及应用实例分析

       privatestaticobjectAddElement(IDictionarycollection,objectkey,objectnewValue)
       {
           objectelement=collection[key];
           collection[key]=newValue;
           returnelement;
       }
 
       privateintGetTermIndex(stringterm)
       {
           objectindex=_wordsIndex[term];
           if(index==null)return-1;
           return(int)index;
       }
 
       privatevoidMyInit()
       {
           _terms=GenerateTerms(_docs);
           _numTerms=_terms.Count;
 
           _maxTermFreq=newint[_numDocs];
           _docFreq=newint[_numTerms];
           _termFreq=newint[_numTerms][];
           _termWeight=newfloat[_numTerms][];
 
           for(inti=0;i<_terms.Count;i++)           
           {
               _termWeight[i]=newfloat[_numDocs];
               _termFreq[i]=newint[_numDocs];
 
               AddElement(_wordsIndex,_terms[i],i);           
           }
           
           GenerateTermFrequency();
           GenerateTermWeight();           
       }
       
       privatefloatLog(floatnum)
       {
           return(float)Math.Log(num);//log2
       }
 
       privatevoidGenerateTermFrequency()
       {
           for(inti=0;i<_numDocs ;i++)
           {                               
               stringcurDoc=_docs[i];
               IDictionaryfreq=GetWordFrequency(curDoc);
               IDictionaryEnumeratorenums=freq.GetEnumerator();
               _maxTermFreq[i]=int.MinValue;
               while(enums.MoveNext())
               {
                   stringword=(string)enums.Key;
                   intwordFreq=(int)enums.Value;
                   inttermIndex=GetTermIndex(word);
 
                   _termFreq[termIndex][i]=wordFreq;
                   _docFreq[termIndex]++;
 
                   if(wordFreq>_maxTermFreq[i])_maxTermFreq[i]=wordFreq;                   
               }
           }
       }

       privatevoidGenerateTermWeight()
       {           
           for(inti=0;i<_numTerms  ;i++)
           {
               for(intj=0;j<_numDocs;j++)               
                   _termWeight[i][j]=ComputeTermWeight(i,j);
           }
       }
 
       privatefloatGetTermFrequency(intterm,intdoc)
       {           
           intfreq=_termFreq[term][doc];
           intmaxfreq=_maxTermFreq[doc];           
           
           return((float)freq/(float)maxfreq);
       }
 
       privatefloatGetInverseDocumentFrequency(intterm)
       {
           intdf=_docFreq[term];
           returnLog((float)(_numDocs)/(float)df);
       }
 
       privatefloatComputeTermWeight(intterm,intdoc)
       {
           floattf=GetTermFrequency(term,doc);
           floatidf=GetInverseDocumentFrequency(term);
           returntf*idf;
       }
       
       private float[]GetTermVector(intdoc)
       {
           float[]w=newfloat[_numTerms];
           for(inti=0;i<_numTerms;i++)
               w[i]=_termWeight[i][doc];
           returnw;
       }
 
       publicfloatGetSimilarity(intdoc_i,intdoc_j)
       {
           float[]vector1=GetTermVector(doc_i);
           float[]vector2=GetTermVector(doc_j);
           returnTermVector.ComputeCosineSimilarity(vector1,vector2);
       }
       
       privateIDictionaryGetWordFrequency(stringinput)
       {
           stringconvertedInput=input.ToLower();
           Tokenisertokenizer=newTokeniser();
           String[]words=tokenizer.Partition(convertedInput);
           Array.Sort(words);
           
           String[]distinctWords=GetDistinctWords(words);
                       
           IDictionaryresult=newHashtable();
           for(inti=0;i<distinctWords.Length;i++)
           {
               objecttmp;
               tmp=CountWords(distinctWords[i],words);
               result[distinctWords[i]]=tmp;
           }
           returnresult;
       }               
               
       privatestring[]GetDistinctWords(String[]input)
       {               
           if(input==null)           
               returnnewstring[0];           
           else
           {
               ArrayListlist=newArrayList();
               
               for(inti=0;i<input.Length;i++)
                   if(!list.Contains(input[i]))//N-GRAMSIMILARITY?
                       list.Add(input[i]);
               returnTokeniser.ArrayListToArray(list);
           }
       }

       privateintCountWords(stringword,string[]words)
       {
           intitemIdx=Array.BinarySearch(words,word);
           
           if(itemIdx>0)           
               while(itemIdx>0&&words[itemIdx].Equals(word))
                   itemIdx--;               
           intcount=0;
           while(itemIdx<words.Length&&itemIdx>=0)
           {
               if(words[itemIdx].Equals(word))count++;
               itemIdx++;
               if(itemIdx<words.Length)               
                   if(!words[itemIdx].Equals(word))break;
           }
           returncount;
       }               
}