You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
107 lines
3.3 KiB
107 lines
3.3 KiB
from .typedefs cimport attr_t, hash_t, flags_t, len_t, tag_t
|
|
from .attrs cimport attr_id_t
|
|
from .attrs cimport ID, ORTH, LOWER, NORM, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER, LANG
|
|
|
|
from .structs cimport LexemeC, SerializedLexemeC
|
|
from .strings cimport StringStore
|
|
from .vocab cimport Vocab
|
|
|
|
from numpy cimport ndarray
|
|
|
|
|
|
cdef LexemeC EMPTY_LEXEME
|
|
|
|
cdef class Lexeme:
|
|
cdef LexemeC* c
|
|
cdef readonly Vocab vocab
|
|
cdef readonly attr_t orth
|
|
|
|
@staticmethod
|
|
cdef inline Lexeme from_ptr(LexemeC* lex, Vocab vocab, int vector_length):
|
|
cdef Lexeme self = Lexeme.__new__(Lexeme, vocab, lex.orth)
|
|
self.c = lex
|
|
self.vocab = vocab
|
|
self.orth = lex.orth
|
|
|
|
@staticmethod
|
|
cdef inline SerializedLexemeC c_to_bytes(const LexemeC* lex) nogil:
|
|
cdef SerializedLexemeC lex_data
|
|
buff = <const unsigned char*>&lex.flags
|
|
end = <const unsigned char*>&lex.sentiment + sizeof(lex.sentiment)
|
|
for i in range(sizeof(lex_data.data)):
|
|
lex_data.data[i] = buff[i]
|
|
return lex_data
|
|
|
|
@staticmethod
|
|
cdef inline void c_from_bytes(LexemeC* lex, SerializedLexemeC lex_data) nogil:
|
|
buff = <unsigned char*>&lex.flags
|
|
end = <unsigned char*>&lex.sentiment + sizeof(lex.sentiment)
|
|
for i in range(sizeof(lex_data.data)):
|
|
buff[i] = lex_data.data[i]
|
|
|
|
@staticmethod
|
|
cdef inline void set_struct_attr(LexemeC* lex, attr_id_t name, attr_t value) nogil:
|
|
if name < (sizeof(flags_t) * 8):
|
|
Lexeme.c_set_flag(lex, name, value)
|
|
elif name == ID:
|
|
lex.id = value
|
|
elif name == LOWER:
|
|
lex.lower = value
|
|
elif name == NORM:
|
|
lex.norm = value
|
|
elif name == SHAPE:
|
|
lex.shape = value
|
|
elif name == PREFIX:
|
|
lex.prefix = value
|
|
elif name == SUFFIX:
|
|
lex.suffix = value
|
|
elif name == CLUSTER:
|
|
lex.cluster = value
|
|
elif name == LANG:
|
|
lex.lang = value
|
|
|
|
@staticmethod
|
|
cdef inline attr_t get_struct_attr(const LexemeC* lex, attr_id_t feat_name) nogil:
|
|
if feat_name < (sizeof(flags_t) * 8):
|
|
if Lexeme.c_check_flag(lex, feat_name):
|
|
return 1
|
|
else:
|
|
return 0
|
|
elif feat_name == ID:
|
|
return lex.id
|
|
elif feat_name == ORTH:
|
|
return lex.orth
|
|
elif feat_name == LOWER:
|
|
return lex.lower
|
|
elif feat_name == NORM:
|
|
return lex.norm
|
|
elif feat_name == SHAPE:
|
|
return lex.shape
|
|
elif feat_name == PREFIX:
|
|
return lex.prefix
|
|
elif feat_name == SUFFIX:
|
|
return lex.suffix
|
|
elif feat_name == LENGTH:
|
|
return lex.length
|
|
elif feat_name == CLUSTER:
|
|
return lex.cluster
|
|
elif feat_name == LANG:
|
|
return lex.lang
|
|
else:
|
|
return 0
|
|
|
|
@staticmethod
|
|
cdef inline bint c_check_flag(const LexemeC* lexeme, attr_id_t flag_id) nogil:
|
|
cdef flags_t one = 1
|
|
if lexeme.flags & (one << flag_id):
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
@staticmethod
|
|
cdef inline bint c_set_flag(LexemeC* lex, attr_id_t flag_id, bint value) nogil:
|
|
cdef flags_t one = 1
|
|
if value:
|
|
lex.flags |= one << flag_id
|
|
else:
|
|
lex.flags &= ~(one << flag_id)
|