| fastText d00d36476b15 Fast text processing tool/library

fasttext.h
Go to the documentation of this file.
1
10 #ifndef FASTTEXT_FASTTEXT_H
11 #define FASTTEXT_FASTTEXT_H
12
13 #define FASTTEXT_VERSION 11 /* Version 1a */
14 #define FASTTEXT_FILEFORMAT_MAGIC_INT32 793712314
15
16 #include <time.h>
17
18 #include <atomic>
19 #include <memory>
20 #include <set>
21
22 #include "args.h"
23 #include "dictionary.h"
24 #include "matrix.h"
25 #include "qmatrix.h"
26 #include "model.h"
27 #include "real.h"
28 #include "utils.h"
29 #include "vector.h"
30
31 namespace fasttext {
32
33 class FastText {
34 private:
35 std::shared_ptr<Args> args_;
36 std::shared_ptr<Dictionary> dict_;
37
38 std::shared_ptr<Matrix> input_;
39 std::shared_ptr<Matrix> output_;
40
41 std::shared_ptr<QMatrix> qinput_;
42 std::shared_ptr<QMatrix> qoutput_;
43
44 std::shared_ptr<Model> model_;
45
46 std::atomic<int64_t> tokenCount;
47 clock_t start;
48 void signModel(std::ostream&);
49 bool checkModel(std::istream&);
50
51 bool quant_;
52
53 public:
54 FastText();
55
56 void getVector(Vector&, const std::string&) const;
57 void saveVectors();
58 void saveOutput();
59 void saveModel();
60 void loadModel(std::istream&);
61 void loadModel(const std::string&);
62 void printInfo(real, real);
63
64 void supervised(Model&, real, const std::vector<int32_t>&,
65 const std::vector<int32_t>&);
66 void cbow(Model&, real, const std::vector<int32_t>&);
67 void skipgram(Model&, real, const std::vector<int32_t>&);
68 std::vector<int32_t> selectEmbeddings(int32_t) const;
69 void quantize(std::shared_ptr<Args>);
70 void test(std::istream&, int32_t);
71 void predict(std::istream&, int32_t, bool);
72 void predict(
73 std::istream&,
74 int32_t,
75 std::vector<std::pair<real, std::string>>&) const;
76 void wordVectors();
77 void sentenceVectors();
78 void ngramVectors(std::string);
79 void textVectors();
80 void printWordVectors();
81 void printSentenceVectors();
82 void precomputeWordVectors(Matrix&);
83 void findNN(const Matrix&, const Vector&, int32_t,
84 const std::set<std::string>&);
85 void nn(int32_t);
86 void analogies(int32_t);
87 void trainThread(int32_t);
88 void train(std::shared_ptr<Args>);
89
90 void loadVectors(std::string);
91 int getDimension() const;
92 };
93
94 }
95 #endif
fasttext::FastText::textVectors
void textVectors()
Definition: fasttext.cc:402
fasttext::FastText::model_
std::shared_ptr< Model > model_
Definition: fasttext.h:44
fasttext::FastText::printSentenceVectors
void printSentenceVectors()
Definition: fasttext.cc:422
fasttext::FastText::supervised
void supervised(Model &, real, const std::vector< int32_t > &, const std::vector< int32_t > &)
Definition: fasttext.cc:257
fasttext::FastText::saveModel
void saveModel()
Definition: fasttext.cc:93
fasttext::FastText::getVector
void getVector(Vector &, const std::string &) const
Definition: fasttext.cc:28
fasttext::FastText::ngramVectors
void ngramVectors(std::string)
Definition: fasttext.cc:388
fasttext::FastText
Definition: fasttext.h:33
fasttext::FastText::loadVectors
void loadVectors(std::string)
Definition: fasttext.cc:551
fasttext::FastText::tokenCount
std::atomic< int64_t > tokenCount
Definition: fasttext.h:46
fasttext::FastText::qoutput_
std::shared_ptr< QMatrix > qoutput_
Definition: fasttext.h:42
fasttext::FastText::findNN
void findNN(const Matrix &, const Vector &, int32_t, const std::set< std::string > &)
Definition: fasttext.cc:443
fasttext::FastText::cbow
void cbow(Model &, real, const std::vector< int32_t > &)
Definition: fasttext.cc:266
fasttext::FastText::selectEmbeddings
std::vector< int32_t > selectEmbeddings(int32_t) const
Definition: fasttext.cc:194
fasttext::FastText::nn
void nn(int32_t)
Definition: fasttext.cc:467
fasttext::FastText::getDimension
int getDimension() const
Definition: fasttext.cc:645
model.h
fasttext
Definition: args.cc:17
fasttext::FastText::FastText
FastText()
Definition: fasttext.cc:26
fasttext::Vector
Definition: vector.h:23
fasttext::FastText::trainThread
void trainThread(int32_t)
Definition: fasttext.cc:511
fasttext::FastText::analogies
void analogies(int32_t)
Definition: fasttext.cc:483
fasttext::FastText::test
void test(std::istream &, int32_t)
Definition: fasttext.cc:297
fasttext::FastText::checkModel
bool checkModel(std::istream &)
Definition: fasttext.cc:72
utils.h
fasttext::FastText::loadModel
void loadModel(std::istream &)
Definition: fasttext.cc:140
fasttext::FastText::start
clock_t start
Definition: fasttext.h:47
fasttext::Model
Definition: model.h:38
fasttext::FastText::quant_
bool quant_
Definition: fasttext.h:51
real.h
fasttext::FastText::printWordVectors
void printWordVectors()
Definition: fasttext.cc:418
fasttext::FastText::train
void train(std::shared_ptr< Args >)
Definition: fasttext.cc:591
fasttext::FastText::saveVectors
void saveVectors()
Definition: fasttext.cc:39
fasttext::FastText::saveOutput
void saveOutput()
Definition: fasttext.cc:55
fasttext::FastText::sentenceVectors
void sentenceVectors()
Definition: fasttext.cc:368
matrix.h
fasttext::FastText::predict
void predict(std::istream &, int32_t, bool)
Definition: fasttext.cc:338
fasttext::FastText::qinput_
std::shared_ptr< QMatrix > qinput_
Definition: fasttext.h:41
fasttext::FastText::output_
std::shared_ptr< Matrix > output_
Definition: fasttext.h:39
fasttext::FastText::input_
std::shared_ptr< Matrix > input_
Definition: fasttext.h:38
vector.h
fasttext::FastText::args_
std::shared_ptr< Args > args_
Definition: fasttext.h:35
fasttext::FastText::precomputeWordVectors
void precomputeWordVectors(Matrix &)
Definition: fasttext.cc:430
args.h
fasttext::real
float real
Definition: real.h:15
fasttext::Matrix
Definition: matrix.h:23
fasttext::FastText::skipgram
void skipgram(Model &, real, const std::vector< int32_t > &)
Definition: fasttext.cc:283
fasttext::FastText::printInfo
void printInfo(real, real)
Definition: fasttext.cc:178
fasttext::FastText::wordVectors
void wordVectors()
Definition: fasttext.cc:359
fasttext::FastText::dict_
std::shared_ptr< Dictionary > dict_
Definition: fasttext.h:36
fasttext::FastText::quantize
void quantize(std::shared_ptr< Args >)
Definition: fasttext.cc:208
fasttext::FastText::signModel
void signModel(std::ostream &)
Definition: fasttext.cc:86
dictionary.h
qmatrix.h