Implement more Po method for merge

Implement file oprerations(entire reading
and rest part of writing)
Convert text to merge/sdf form
Plus cleanup

Change-Id: I6bab9bb202fcdc6d0f2a13f6c44d28e22c688e1e
Reviewed-on: https://gerrit.libreoffice.org/520
Reviewed-by: Andras Timar <atimar@suse.com>
Tested-by: Andras Timar <atimar@suse.com>

6 files changed, 299 insertions, 53 deletions

diff --git a/l10ntools/Executable_localize.mk b/l10ntools/Executable_localize.mk
index 6184a8982bd5..e6e7f1fc2c6e 100644
--- a/l10ntools/Executable_localize.mk
+++ b/l10ntools/Executable_localize.mk
@@ -33,6 +33,7 @@ $(eval $(call gb_Executable_set_include,localize,\
 
 $(eval $(call gb_Executable_use_libraries,localize,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_use_static_libraries,localize,\
diff --git a/l10ntools/Executable_renewpo.mk b/l10ntools/Executable_renewpo.mk
index 4872e8a2bb77..5aa036a40100 100644
--- a/l10ntools/Executable_renewpo.mk
+++ b/l10ntools/Executable_renewpo.mk
@@ -18,6 +18,7 @@ $(eval $(call gb_Executable_set_include,renewpo,\
 
 $(eval $(call gb_Executable_use_libraries,renewpo,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_use_static_libraries,renewpo,\
diff --git a/l10ntools/StaticLibrary_transex.mk b/l10ntools/StaticLibrary_transex.mk
index 21f50e572ef6..82e20a012c5f 100644
--- a/l10ntools/StaticLibrary_transex.mk
+++ b/l10ntools/StaticLibrary_transex.mk
@@ -32,6 +32,8 @@ $(eval $(call gb_StaticLibrary_set_include,transex,\
     $$(INCLUDE) \
 ))
 
+$(eval $(call gb_StaticLibrary_use_sdk_api,transex))
+
 $(eval $(call gb_StaticLibrary_add_exception_objects,transex,\
     l10ntools/source/export2 \
     l10ntools/source/merge \
diff --git a/l10ntools/inc/po.hxx b/l10ntools/inc/po.hxx
index c03baf8dc829..df3004c1fe5e 100644
--- a/l10ntools/inc/po.hxx
+++ b/l10ntools/inc/po.hxx
@@ -31,6 +31,14 @@ public:
                         GenPoEntry();
     virtual             ~GenPoEntry();
 
+    virtual OString     getWhiteSpace() const   { return m_sWhiteSpace; }
+    virtual OString     getExtractCom() const   { return m_sExtractCom; }
+    virtual OString     getReference() const    { return m_sReference; }
+    virtual OString     getContext() const      { return m_sContext; }
+    virtual OString     getUnTransStr() const   { return m_sUnTransStr; }
+    virtual OString     getTransStr() const     { return m_sTransStr; }
+    virtual bool        getFuzzy() const        { return m_bFuzzy; }
+
     virtual void        setWhiteSpace(const OString& rWhiteSpace)
                         { m_sWhiteSpace = rWhiteSpace; }
     virtual void        setExtractCom(const OString& rExtractCom)
@@ -46,7 +54,9 @@ public:
     virtual void        setFuzzy(bool bFuzzy)
                         { m_bFuzzy = bFuzzy; }
     virtual void        genKeyId();
-    virtual void        writeToFile(std::ofstream& io_rOFStream);
+
+    virtual void        writeToFile(std::ofstream& rOFStream);
+    virtual void        readFromFile(std::ifstream& rIFStream);
 };
 
 class PoEntry: public GenPoEntry
@@ -67,13 +77,19 @@ private:
 
 public:
 
+                        PoEntry();
                         PoEntry(const OString& i_rSDFLine,
-                            const TYPE eType = TTEXT);
+                                const TYPE eType = TTEXT);
     virtual             ~PoEntry();
 
+    virtual OString     getUnTransStr() const;
+    virtual OString     getTransStr() const;
     virtual void        setUnTransStr(const OString& rUnTransStr);
     virtual void        setTransStr(const OString& rTransStr);
 
+    virtual void        writeToFile(std::ofstream& rOFStream);
+    virtual void        readFromFile(std::ifstream& rIFStream);
+
 };
 
 class PoHeader: public GenPoEntry
diff --git a/l10ntools/source/po.cxx b/l10ntools/source/po.cxx
index 6848d7e3680f..a82a88b186f4 100644
--- a/l10ntools/source/po.cxx
+++ b/l10ntools/source/po.cxx
@@ -7,11 +7,20 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-#include "po.hxx"
+#include <com/sun/star/util/SearchOptions.hpp>
+#include <com/sun/star/util/SearchFlags.hpp>
+#include <com/sun/star/i18n/XExtendedTransliteration.hpp>
+#include <regexp/reclass.hxx>
+#include <rtl/ustring.hxx>
+
+#include <string>
+#include <cstring>
 #include <ctime>
 #include <vector>
 #include <boost/crc.hpp>
 
+#include "po.hxx"
+
 #define POESCAPED OString("\\n\\t\\r\\\\\\\"")
 #define POUNESCAPED OString("\n\t\r\\\"")
 
@@ -69,17 +78,17 @@ OString ImplUnEscapeText(const OString& rText,
     return sResult;
 }
 
-//Generate msgctxt, msgid and msgstr strings
-OString ImplGenMsgString(const OString& rSource)
+//Convert a normal string to msg/po output string
+OString ImplGenMsgString(const OString& rString)
 {
-    if ( rSource.isEmpty() )
+    if ( rString.isEmpty() )
         return "\"\"";
 
-    OString sResult = "\"" + rSource + "\"";
+    OString sResult = "\"" + ImplEscapeText(rString) + "\"";
     sal_Int32 nIndex = 0;
     while((nIndex=sResult.indexOf("\\n",nIndex))!=-1)
     {
-        if( sResult.copy(nIndex-1,3) != "\\\\n" )
+        if(sResult.copy(nIndex-1,3)!="\\\\n" && nIndex!=sResult.getLength()-3)
            sResult = sResult.replaceAt(nIndex,2,"\\n\"\n\"");
         ++nIndex;
     }
@@ -90,6 +99,18 @@ OString ImplGenMsgString(const OString& rSource)
     return sResult;
 }
 
+//Convert msg string to normal form
+OString ImplGenNormString(const OString& rString)
+{
+    return ImplUnEscapeText(rString.copy(1,rString.getLength()-2));
+}
+
+//Decide whether a string starts with an other string
+bool ImplStartsWith(const OString& rString,const OString& rStart)
+{
+    return rString.copy(0,rStart.getLength())==rStart;
+}
+
 //Default constructor
 GenPoEntry::GenPoEntry()
     : m_sWhiteSpace( OString() )
@@ -115,29 +136,79 @@ void GenPoEntry::genKeyId()
 }
 
 //Write to file
-void GenPoEntry::writeToFile(std::ofstream& io_rOFStream)
+void GenPoEntry::writeToFile(std::ofstream& rOFStream)
 {
     if ( !m_sWhiteSpace.isEmpty() )
-        io_rOFStream << m_sWhiteSpace.getStr();
+        rOFStream << m_sWhiteSpace.getStr();
     if ( !m_sExtractCom.isEmpty() )
-        io_rOFStream << "#. " << m_sExtractCom.getStr() << std::endl;
+        rOFStream << "#. " << m_sExtractCom.getStr() << std::endl;
     if ( !m_sKeyId.isEmpty() )
-        io_rOFStream << "#. " << m_sKeyId.getStr() << std::endl;
+        rOFStream << "#. " << m_sKeyId.getStr() << std::endl;
     if ( !m_sReference.isEmpty() )
-        io_rOFStream << "#: " << m_sReference.getStr() << std::endl;
+        rOFStream << "#: " << m_sReference.getStr() << std::endl;
     if ( m_bFuzzy )
-        io_rOFStream << "#, fuzzy" << std::endl;
+        rOFStream << "#, fuzzy" << std::endl;
     if ( !m_sContext.isEmpty() )
-        io_rOFStream << "msgctxt "
-                     << ImplGenMsgString(m_sContext).getStr() << std::endl;
-    io_rOFStream << "msgid "
-                 << ImplGenMsgString(ImplEscapeText(m_sUnTransStr)).getStr()
-                 << std::endl;
-    io_rOFStream << "msgstr "
-                 << ImplGenMsgString(ImplEscapeText(m_sTransStr)).getStr()
-                 << std::endl;
+        rOFStream << "msgctxt "
+                  << ImplGenMsgString(m_sContext).getStr() << std::endl;
+    rOFStream << "msgid "
+              << ImplGenMsgString(m_sUnTransStr).getStr() << std::endl;
+    rOFStream << "msgstr "
+              << ImplGenMsgString(m_sTransStr).getStr() << std::endl;
 }
 
+//Read from file
+void GenPoEntry::readFromFile(std::ifstream& rIFStream)
+{
+    m_sWhiteSpace = "\n";
+    OString* pLastMsg = 0;
+    std::string sTemp;
+    getline(rIFStream,sTemp);
+    while(!rIFStream.eof())
+    {
+        OString sLine = OString(sTemp.data(),sTemp.length());
+        if (ImplStartsWith(sLine,"#. "))
+        {
+            if (sLine.getLength()==7)
+                m_sKeyId = sLine.copy(3);
+            else
+                m_sExtractCom = sLine.copy(3);
+        }
+        else if (ImplStartsWith(sLine,"#: "))
+        {
+            m_sReference = sLine.copy(3);
+        }
+        else if (ImplStartsWith(sLine,"#, fuzzy"))
+        {
+            m_bFuzzy = true;
+        }
+        else if (ImplStartsWith(sLine,"msgctxt "))
+        {
+            m_sContext = ImplGenNormString(sLine.copy(8));
+            pLastMsg = &m_sContext;
+        }
+        else if (ImplStartsWith(sLine,"msgid "))
+        {
+            m_sUnTransStr = ImplGenNormString(sLine.copy(6));
+            pLastMsg = &m_sUnTransStr;
+        }
+        else if (ImplStartsWith(sLine,"msgstr "))
+        {
+            m_sTransStr = ImplGenNormString(sLine.copy(7));
+            pLastMsg = &m_sTransStr;
+        }
+        else if (ImplStartsWith(sLine,"\"") && pLastMsg)
+        {
+            *pLastMsg += ImplGenNormString(sLine);
+        }
+        else
+            break;
+        getline(rIFStream,sTemp);
+    }
+    if (m_sKeyId.isEmpty())
+        genKeyId();
+ }
+
 //Class PoEntry
 
 //Split string at the delimiter char
@@ -159,8 +230,122 @@ void ImplSplitAt(const OString& rSource, const sal_Char nDelimiter,
     o_vParts.push_back(rSource.copy(nLastSplit));
 }
 
+//Unescape sdf string
+OString ImplUnEscapeSDFText(const OString& rText,const bool bHelpText = false)
+{
+    if ( bHelpText )
+        return ImplUnEscapeText(rText,"\\<\\>\\\"\\\\","<>\"\\");
+    else
+        return ImplUnEscapeText(rText,"\\n\\t\\r","\n\t\r");
+}
+
+//Miminize the length of the regular expression result
+void ImplMinimize(const OUString& rText, Regexpr& io_rRegExp, re_registers& io_rRegs)
+{
+    re_registers aPrevRegs;
+    const sal_Int32 nStart = io_rRegs.start[0];
+    do
+    {
+        const OUString sTemp = rText.copy(0,io_rRegs.end[0]-1);
+        memcpy(static_cast<void*>(&aPrevRegs), static_cast<void*>(&io_rRegs),
+               sizeof(re_registers));
+        memset(static_cast<void*>(&io_rRegs), 0, sizeof(re_registers));
+        io_rRegExp.set_line(sTemp.getStr(),sTemp.getLength());
+        io_rRegExp.re_search(&io_rRegs,nStart);
+    } while(io_rRegs.num_of_match);
+
+    memcpy(static_cast<void*>(&io_rRegs),static_cast<void*>(&aPrevRegs),
+           sizeof(re_registers));
+    io_rRegExp.set_line(rText.getStr(),rText.getLength());
+}
+
+//Find all special tag in a string using a regular expression
+void ImplFindAllTag(const OString& rText,std::vector<OString>& o_vFoundTags)
+{
+    ::com::sun::star::util::SearchOptions aOptions;
+    aOptions.algorithmType = ::com::sun::star::util::SearchAlgorithms_REGEXP;
+    aOptions.searchFlag = ::com::sun::star::util::SearchFlags::NORM_WORD_ONLY;
+    aOptions.searchString = "<[/]?[a-z_\\-]+(| +[a-z]+=\".*\") *[/]?>";
+    ::com::sun::star::uno::Reference<
+         ::com::sun::star::i18n::XExtendedTransliteration > xTrans;
+
+    Regexpr aRegExp(aOptions,xTrans);
+    const OUString sTemp(OStringToOUString(rText,RTL_TEXTENCODING_UTF8));
+    aRegExp.set_line(sTemp.getStr(),sTemp.getLength());
+
+    re_registers aRegs;
+    memset(static_cast<void*>(&aRegs), 0, sizeof(re_registers));
+    sal_Int32 nStart = 0;
+    o_vFoundTags.resize(0);
+    aRegExp.re_search(&aRegs,nStart);
+    while(aRegs.num_of_match)
+    {
+        ImplMinimize(sTemp,aRegExp,aRegs);
+        o_vFoundTags.push_back(
+                rText.copy(aRegs.start[0],aRegs.end[0]-aRegs.start[0]));
+        nStart = aRegs.end[0];
+        memset(static_cast<void*>(&aRegs), 0, sizeof(re_registers));
+        aRegExp.re_search(&aRegs,nStart);
+    }
+}
+
+//Escape special tags
+OString ImplEscapeTags(const OString& rText)
+{
+    typedef std::vector<OString> StrVec;
+    const StrVec vTagsForEscape =
+                  { "ahelp", "link", "item", "emph", "defaultinline",
+                    "switchinline", "caseinline", "variable",
+                    "bookmark_value", "image", "embedvar", "alt" };
+    StrVec vFoundTags;
+    ImplFindAllTag(rText,vFoundTags);
+    OString sResult = rText;
+    for(StrVec::const_iterator pFound  = vFoundTags.begin();
+        pFound != vFoundTags.end(); ++pFound)
+    {
+        bool bEscapeThis = false;
+        for(StrVec::const_iterator pEscape = vTagsForEscape.begin();
+            pEscape != vTagsForEscape.end(); ++pEscape)
+        {
+            if (ImplStartsWith(*pFound,"<" + *pEscape) ||
+                *pFound == "</" + *pEscape + ">")
+            {
+                bEscapeThis = true;
+                break;
+            }
+        }
+        if (bEscapeThis || *pFound=="<br/>" || *pFound =="<help-id-missing/>")
+        {
+            OString sToReplace = "\\<" + pFound->copy(1,pFound->getLength()-2).
+                                           replaceAll("\"","\\\"") + "\\>";
+            sResult = sResult.replaceAll(*pFound, sToReplace);
+        }
+    }
+    return sResult;
+}
+
+//Escape to get sdf/merge string
+OString ImplEscapeSDFText(const OString& rText,const bool bHelpText = false)
+{
+    if ( bHelpText )
+        return ImplEscapeTags(rText.replaceAll("\\","\\\\"));
+    else
+        return ImplEscapeText(rText,"\n\t\r","\\n\\t\\r");
+}
 
 
+//Default constructor
+PoEntry::PoEntry()
+    : GenPoEntry()
+    , m_sSourceFile( OString() )
+    , m_sGroupId( OString() )
+    , m_sLocalId( OString() )
+    , m_sResourceType( OString() )
+    , m_eType( TTEXT )
+    , m_sHelpText( OString() )
+{
+}
+
 //Construct PoEntry from sdfline
 PoEntry::PoEntry(const OString& rSDFLine, const TYPE eType)
     : m_sSourceFile( OString() )
@@ -168,39 +353,20 @@ PoEntry::PoEntry(const OString& rSDFLine, const TYPE eType)
     , m_sLocalId( OString() )
     , m_sResourceType(OString() )
     , m_eType( TTEXT )
+    , m_sHelpText( OString() )
 {
-    setWhiteSpace("\n");
     std::vector<OString> vParts;
     ImplSplitAt(rSDFLine,'\t',vParts);
     if(vParts.size()!=15) throw;
 
     m_sSourceFile = vParts[SOURCEFILE].
                         copy(vParts[SOURCEFILE].lastIndexOf("\\")+1);
-    setReference(m_sSourceFile);
+    m_sResourceType = vParts[RESOURCETYPE];
     m_sGroupId = vParts[GROUPID];
     m_sLocalId = vParts[LOCALID];
-    m_sResourceType = vParts[RESOURCETYPE];
     m_eType = eType;
     m_sHelpText = vParts[HELPTEXT];
-
-    OString sContext = m_sGroupId + "\\n" +
-                       (m_sLocalId.isEmpty() ? "" : m_sLocalId + "\\n") +
-                       m_sResourceType;
-    switch(eType){
-    case TTEXT:
-        sContext += ".text"; break;
-    case TQUICKHELPTEXT:
-        sContext += ".quickhelptext"; break;
-    case TTITLE:
-        sContext += ".title"; break;
-    default:
-        throw; break;
-    }
-    setContext(sContext);
-    setExtractCom(m_sHelpText);
-
     setUnTransStr(vParts[eType]);
-    genKeyId();
 }
 
 //Destructor
@@ -208,16 +374,22 @@ PoEntry::~PoEntry()
 {
 }
 
-//Unescape sdf text
-OString ImplUnEscapeSDFText(const OString& rText,const bool bHelpText = false)
+//Get translation string in sdf/merge format
+OString PoEntry::getUnTransStr() const
 {
-    if ( bHelpText )
-        return ImplUnEscapeText(rText,"\\<\\>\\\"\\\\","<>\"\\");
-    else
-        return ImplUnEscapeText(rText,"\\n\\t\\r","\n\t\r");
+    return ImplEscapeSDFText(GenPoEntry::getUnTransStr(),
+                             m_sSourceFile.endsWith(".xhp"));
+}
+
+//Get translated string in sdf/merge format
+OString PoEntry::getTransStr() const
+{
+    return ImplEscapeSDFText(GenPoEntry::getTransStr(),
+                             m_sSourceFile.endsWith(".xhp"));
+
 }
 
-//Set translation text when input is in sdf format
+//Set translation string when input is in sdf format
 void PoEntry::setUnTransStr(const OString& rUnTransStr)
 {
     GenPoEntry::setUnTransStr(
@@ -225,7 +397,7 @@ void PoEntry::setUnTransStr(const OString& rUnTransStr)
                         rUnTransStr,m_sSourceFile.endsWith(".xhp")));
 }
 
-//Set translated text when input is in sdf format
+//Set translated string when input is in sdf format
 void PoEntry::setTransStr(const OString& rTransStr)
 {
     GenPoEntry::setTransStr(
@@ -233,6 +405,60 @@ void PoEntry::setTransStr(const OString& rTransStr)
                         rTransStr,m_sSourceFile.endsWith(".xhp")));
 }
 
+//Write to file
+void PoEntry::writeToFile(std::ofstream& rOFStream)
+{
+    setWhiteSpace("\n");
+    setExtractCom(m_sHelpText);
+    setReference(m_sSourceFile);
+
+    OString sContext = m_sGroupId + "\n" +
+                       (m_sLocalId.isEmpty() ? "" : m_sLocalId + "\n") +
+                       m_sResourceType;
+    switch(m_eType){
+    case TTEXT:
+        sContext += ".text"; break;
+    case TQUICKHELPTEXT:
+        sContext += ".quickhelptext"; break;
+    case TTITLE:
+        sContext += ".title"; break;
+    default:
+        throw; break;
+    }
+    setContext(sContext);
+    genKeyId();
+    GenPoEntry::writeToFile(rOFStream);
+}
+
+
+//Read from file
+void PoEntry::readFromFile(std::ifstream& rIFStream)
+{
+    GenPoEntry::readFromFile(rIFStream);
+    m_sSourceFile = getReference();
+
+    OString sContext =  getContext();
+    m_sGroupId = sContext.getToken(0,'\n');
+
+    if (sContext.indexOf('\n')==sContext.lastIndexOf('\n'))
+        m_sResourceType = sContext.getToken(1,'\n').getToken(0,'.');
+    else
+    {
+        m_sLocalId = sContext.getToken(1,'\n');
+        m_sResourceType = sContext.getToken(2,'\n').getToken(0,'.');
+    }
+    if (sContext.endsWith(".text"))
+        m_eType = TTEXT;
+    else if (sContext.endsWith(".quickhelptext"))
+        m_eType = TQUICKHELPTEXT;
+    else if (sContext.endsWith(".title"))
+        m_eType = TTITLE;
+    else
+        throw;
+
+    m_sHelpText = getExtractCom();
+}
+
 //Class PoHeader
 
 //Get actual time in "YEAR-MO-DA HO:MI+ZONE" form
diff --git a/l10ntools/source/renewpo.cxx b/l10ntools/source/renewpo.cxx
index a6e127356118..7e567f07d52e 100644
--- a/l10ntools/source/renewpo.cxx
+++ b/l10ntools/source/renewpo.cxx
@@ -63,7 +63,7 @@ void HandleLanguage(struct dirent* pLangEntry, const OString& rPath,
     //Generate and open sdf
     cout << "Process start with language: " <<  LangEntryName.getStr() << endl;
     system( (rpo2loPath +
-            " -i " + rPath.getStr() + LangEntryName +
+            " -i " + rPath + "/" + LangEntryName +
             " -o " + SDFFileName +
             " -l " + LangEntryName +
             " -t " + rSDFPath).getStr());
@@ -77,7 +77,7 @@ void HandleLanguage(struct dirent* pLangEntry, const OString& rPath,
     while(!aSDFInput.eof())
     {
         OString sActUnTrans = sLine;
-        OString sPath = rPath + LangEntryName;
+        OString sPath = rPath + "/"+ LangEntryName;
         OString sActSourcePath = GetPath(sPath,sActUnTrans);
         //Make new po file, copy header with some changes
         if (!aOutPut.is_open())