1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
/* This file is part of Strigi Desktop Search
*
* Copyright (C) 2007 Flavio Castelli <flavio.castelli@gmail.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public License
* along with this library; see the file COPYING.LIB. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "txtlineanalyzer.h"
#include <strigi/strigiconfig.h>
#include <strigi/analysisresult.h>
#include <strigi/fieldtypes.h>
using namespace std;
using namespace Strigi;
// AnalyzerFactory
void
TxtLineAnalyzerFactory::registerFields(FieldRegister& reg) {
totalLinesField = reg.registerField(
"http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#lineCount");
totalWordsField = reg.registerField(
"http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#wordCount");
totalCharactersField = reg.registerField(
"http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#characterCount");
maxLineLengthField = reg.registerField(
"http://strigi.sf.net/ontologies/homeless#maxLineLength");
//TODO: this field should receive a proper name. What is meant is whether
// the file uses uses unix (\n), dos (\r\n) or mac (\r) line endings
formatField = reg.registerField(
"http://strigi.sf.net/ontologies/homeless#lineEndingFormat");
typeField = reg.typeField;
addField(totalLinesField);
addField(totalWordsField);
addField(totalCharactersField);
addField(maxLineLengthField);
addField(formatField);
addField(typeField);
}
// Analyzer
void
TxtLineAnalyzer::startAnalysis(AnalysisResult* i) {
analysisResult = i;
totalLines = 0;
totalWords = 0;
totalCharacters = 0;
maxLineLength = 0;
dos = false;
ready = false;
}
void
TxtLineAnalyzer::handleLine(const char* data, uint32_t length) {
bool inWord = false;
totalLines++;
totalCharacters += length;
if (maxLineLength < length)
maxLineLength = length;
// instead of using regexp use this elementary solution
for (unsigned int i = 0; i < length; i++) {
bool spacer = true;
if (isspace(data[i]) == 0)
spacer = false;
if (!spacer && !inWord) {// beginning of a word
totalWords++;
inWord = true;
}
//else if (!spacer && inWord) {/*inside word, do nothing*/}
else if (spacer && inWord)
inWord = false;
}
//TODO: by now it isn't possible to detect mac formatting
//endline should be just '\r'. I don't know if it is still true with latest
// versions of OSX, I have tried with tiger and I got a standard unix file.
if (length > 0 && data[length-1] == '\r')
dos = true;
}
void
TxtLineAnalyzer::endAnalysis(bool complete) {
// we assume all cpp files must have includes
if (complete) {
analysisResult->addValue(factory->totalWordsField, (int32_t)totalWords);
analysisResult->addValue(factory->totalCharactersField, (int32_t)totalCharacters);
analysisResult->addValue(factory->totalLinesField, (int32_t)totalLines);
/* //FIXME: either get rid of this or replace with NIE equivalent
analysisResult->addValue(factory->maxLineLengthField, (int32_t)maxLineLength);
if (dos)
analysisResult->addValue(factory->formatField, "DOS");
else
analysisResult->addValue(factory->formatField, "UNIX");
*/
analysisResult->addValue(factory->typeField, "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#TextDocument");
}
ready = true;
}
bool
TxtLineAnalyzer::isReadyWithStream() {
return ready;
}
//Factory
class Factory : public AnalyzerFactoryFactory {
public:
list<StreamLineAnalyzerFactory*>
streamLineAnalyzerFactories() const {
list<StreamLineAnalyzerFactory*> af;
af.push_back(new TxtLineAnalyzerFactory());
return af;
}
};
STRIGI_ANALYZER_FACTORY(Factory)
|
