Spaces:
Sleeping
Sleeping
File size: 2,121 Bytes
d916065 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 |
# Natural Language Toolkit: Toolbox Reader
#
# Copyright (C) 2001-2023 NLTK Project
# Author: Greg Aumann <[email protected]>
# Stuart Robinson <[email protected]>
# Steven Bird <[email protected]>
# URL: <https://www.nltk.org/>
# For license information, see LICENSE.TXT
"""
Module for reading, writing and manipulating
Toolbox databases and settings fileids.
"""
from nltk.corpus.reader.api import *
from nltk.corpus.reader.util import *
from nltk.toolbox import ToolboxData
class ToolboxCorpusReader(CorpusReader):
def xml(self, fileids, key=None):
return concat(
[
ToolboxData(path, enc).parse(key=key)
for (path, enc) in self.abspaths(fileids, True)
]
)
def fields(
self,
fileids,
strip=True,
unwrap=True,
encoding="utf8",
errors="strict",
unicode_fields=None,
):
return concat(
[
list(
ToolboxData(fileid, enc).fields(
strip, unwrap, encoding, errors, unicode_fields
)
)
for (fileid, enc) in self.abspaths(fileids, include_encoding=True)
]
)
# should probably be done lazily:
def entries(self, fileids, **kwargs):
if "key" in kwargs:
key = kwargs["key"]
del kwargs["key"]
else:
key = "lx" # the default key in MDF
entries = []
for marker, contents in self.fields(fileids, **kwargs):
if marker == key:
entries.append((contents, []))
else:
try:
entries[-1][-1].append((marker, contents))
except IndexError:
pass
return entries
def words(self, fileids, key="lx"):
return [contents for marker, contents in self.fields(fileids) if marker == key]
def demo():
pass
if __name__ == "__main__":
demo()
|