##############################################################################
#
# Copyright (c) 2003-2004 Hajime Nakagami<nakagami@da2.so-net.ne.jp>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
# 3. The name of the author may not be used to endorse or promote products 
#    derived from this software without specific prior written permission. 
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
##############################################################################

from Products.ZCTextIndex.ISplitter import ISplitter
from Products.ZCTextIndex.PipelineFactory import element_factory

import re
from types import UnicodeType
from SplitJapanese import splitJapanese

class ejSplitter:
    __implements__ = ISplitter
    def process(self, text, glob=0):
        splat = []
        for t in text:
            splat += self._split(t, glob)
        return splat

    def processGlob(self, text):
        return self.process(text, glob=1)

    def _split(self, text, glob):
        text = self._convert(text)
        remove = [r"<[^<>]*>",
                  r"&[A-Za-z]+;"]
        for pat in remove:
            text = re.sub(pat, " ", text)
        return splitJapanese(text, glob)

    def _convert(self, text):
        if type(text) != UnicodeType:
            return unicode(text, 'utf-8')
        else:
            return text

class ejSplitterEUC_JP(ejSplitter):
    def _convert(self, text):
        if type(text) != UnicodeType:
            return unicode(text, 'euc-jp', 'ignore')
        else:
            return text

element_factory.registerFactory('Word Splitter',
                                'ejSplitter for Japanese(UTF-8)',
                                ejSplitter)

element_factory.registerFactory('Word Splitter',
                                'ejSplitter for Japanese(euc-jp)',
                                ejSplitterEUC_JP)

