NET中文分词.docx
- 文档编号:30658185
- 上传时间:2023-08-19
- 格式:DOCX
- 页数:26
- 大小:27.60KB
NET中文分词.docx
《NET中文分词.docx》由会员分享,可在线阅读,更多相关《NET中文分词.docx(26页珍藏版)》请在冰豆网上搜索。
NET中文分词
.NET中文分词
分类:
搜索引擎2010-12-0323:
10 148人阅读 评论(0) 收藏 举报
这两天因为需要就研究了.net环境下的中文分词,发现在目前的最高2.3.1L版本下中文分词效果不好,他自带了一些分词器,均没有什么效果,出来单词切分,连JCK二分法都不提供,更谈不上基于词典的分词了。
下面我发俩段代码,前一段是我通过写一个二分法类来实现双字切分,后一段是通过调用肖波的分词器,自己又写了一个.net环境接口来实现基于词典的分词,以供大家学习之用……
代码段一
[c-sharp] viewplaincopy
1.using System;
2.using System.Collections.Generic;
3.using System.Linq;
4.using System.Text;
5.using Lucene.Net.Analysis.Standard;
6.using Lucene.Net.Analysis;
7.using System.IO;
8.using System.Net;
9.using System.Collections;
10.
11.namespace erfenfa
12.
13.{
14.
15. private class filter
16.
17. {
18.
19. public static void main(string [ ]arg)
20. {
21.
22. string str="我们都是好孩子,即使你不同意,但又能如何,是不?
";
23. TextReader tr = new StringReader(str );
24. CJKAnalyzer wa = new CJKAnalyzer();
25. TokenStream ts = wa.TokenStream("*",tr);
26. Token t = null;
27. while ((t = ts.Next()) !
= null)
28. {
29. messagebox.show(t.TermText().ToString());//实现循环输出
30.
31. }
32.
33.
34. }
35.
36. }
37.
38. public class CJKAnalyzer:
Analyzer
39. {
40.
41. public static string[] STOP_WORDS = {
42. "a", "and", "are", "as", "at", "be",
43. "but", "by", "for", "if", "in",
44. "into", "is", "it", "no", "not",
45. "of", "on", "or", "s", "such", "t",
46. "that", "the", "their", "then",
47. "there", "these", "they", "this",
48. "to", "was", "will", "with", "",
49. "www"
50. };
51.
52.
53. private Hashtable stopTable;
54.
55. public CJKAnalyzer()
56. {
57. stopTable = StopFilter.MakeStopSet(STOP_WORDS);
58. }
59.
60. public CJKAnalyzer(string[] stopWords)
61. {
62. stopTable = StopFilter.MakeStopSet(stopWords);
63. }
64.
65.
66. public override TokenStream TokenStream(string fieldName, TextReader reader)
67. {
68. TokenStream ts=new CJKTokenizer(reader);
69. return new StopFilter(ts, stopTable);
70.
71. }
72. }
73. public class CJKTokenizer:
Tokenizer
74. {
75.
76. private static int MAX_WORD_LEN = 255;
77.
78. private static int IO_BUFFER_SIZE = 256;
79.
80. private int offset = 0;
81.
82.
83. private int bufferIndex = 0;
84.
85. private int dataLen = 0;
86. private char[] buffer = new char[MAX_WORD_LEN];
87.
88. private char[] ioBuffer = new char[IO_BUFFER_SIZE];
89. private string tokenType = "word";
90. private bool preIsTokened = false;
91. public CJKTokenizer(TextReader reader)
92. {
93. input = reader;
94. }
95. public override Token Next()
96. {
97. int length = 0;
98. int start = offset;
99.
100. while (true)
101. {
102.
103. char c;
104. offset++;
105. if (bufferIndex >= dataLen )
106. {
107. if (dataLen==0 || dataLen>=ioBuffer.Length)//Java中read读到最后不会出错,但.Net会,所以此处是为了拦截异常
108. {
109. dataLen = input.Read(ioBuffer,0,ioBuffer.Length);
110. bufferIndex = 0;
111. }
112. else
113. {
114. dataLen=0;
115. }
116. }
117.
118. if (dataLen ==0)
119. {
120. if (length > 0)
121. {
122. if (preIsTokened == true)
123. {
124. length = 0;
125. preIsTokened = false;
126. }
127.
128. break;
129. }
130. else
131. {
132. return null;
133. }
134. }
135. else
136. {
137.
138. c = ioBuffer[bufferIndex++];
139. }
140.
141. if (IsAscii(c) || IsHALFWIDTH_AND_FULLWIDTH_FORMS(c))
142. {
143. if (IsHALFWIDTH_AND_FULLWIDTH_FORMS(c))
144. {
145.
146. int i = (int) c;
147. i = i - 65248;
148. c = (char) i;
149. }
150. #region if the current character is a letter or "_" "+" "#
151.
152. if (char.IsLetterOrDigit(c) || ((c == '_') || (c == '+') || (c == '#')))
153. {
154.
155. if (length == 0)
156. {
157.
158. start = offset - 1;
159. }
160. else if (tokenType == "double")
161. {
162. offset--;
163. bufferIndex--;
164. tokenType = "single";
165.
166. if (preIsTokened == true)
167. {
168.
169. length = 0;
170. preIsTokened = false;
171.
172. break;
173. }
174. else
175. {
176. break;
177. }
178. }
179.
180.
181. buffer[length++] = char.ToLower(c);
182. tokenType = "single";
183.
184. if (length == MAX_WORD_LEN)
185. {
186. break;
187. }
188. }
189. else if (length > 0)
190. {
191. if (preIsTokened == true)
192. {
193. length = 0;
194. preIsTokened = false;
195. }
196. else
197. {
198. break;
199. }
200. }
201. #endregion
202.
203. }
204. else
205. {
206. #region // non-ASCII letter, eg."C1C2C3C4"
207. if (char.IsLetter(c))
208. {
209. if (length == 0)
210. {
211. start = offset - 1;
212. buffer[length++] = c;
213. tokenType = "double";
214. }
215. else
216. {
217. if (tokenType == "single")
218. {
219. offset--;
220. bufferIndex--;
221. break;
222. }
223. else
224. {
225. buffer[length++] = c;
226. tokenType = "double";
227.
228. if (length == 2)
229. {
230. offset--;
231. bufferIndex--;
232. preIsTokened = true;
233.
234. break;
235. }
236. }
237. }
238. }
239. else if (length > 0)
240. {
241. if (preIsTokened == true)
242. {
243.
244. length = 0;
245. preIsTokened = false;
246. }
247. else
248. {
249. break;
250. }
251. }
252. #endregion
253. }
254. }
255. return new Token(new String(bu
- 配套讲稿:
如PPT文件的首页显示word图标,表示该PPT已包含配套word讲稿。双击word图标可打开word文档。
- 特殊限制:
部分文档作品中含有的国旗、国徽等图片,仅作为作品整体效果示例展示,禁止商用。设计者仅对作品中独创性部分享有著作权。
- 关 键 词:
- NET 中文 分词
![提示](https://static.bdocx.com/images/bang_tan.gif)