File indexing completed on 2024-12-22 04:33:42
0001 /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ 0002 /* ***** BEGIN LICENSE BLOCK ***** 0003 * Version: MPL 1.1/GPL 2.0/LGPL 2.1 0004 * 0005 * The contents of this file are subject to the Mozilla Public License Version 0006 * 1.1 (the "License"); you may not use this file except in compliance with 0007 * the License. You may obtain a copy of the License at 0008 * http://www.mozilla.org/MPL/ 0009 * 0010 * Software distributed under the License is distributed on an "AS IS" basis, 0011 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 0012 * for the specific language governing rights and limitations under the 0013 * License. 0014 * 0015 * The Original Code is Mozilla Universal charset detector code. 0016 * 0017 * The Initial Developer of the Original Code is 0018 * Netscape Communications Corporation. 0019 * Portions created by the Initial Developer are Copyright (C) 2001 0020 * the Initial Developer. All Rights Reserved. 0021 * 0022 * Contributor(s): 0023 * Shy Shalom <shooshX@gmail.com> 0024 * 0025 * Alternatively, the contents of this file may be used under the terms of 0026 * either the GNU General Public License Version 2 or later (the "GPL"), or 0027 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), 0028 * in which case the provisions of the GPL or the LGPL are applicable instead 0029 * of those above. If you wish to allow use of your version of this file only 0030 * under the terms of either the GPL or the LGPL, and not to allow others to 0031 * use your version of this file under the terms of the MPL, indicate your 0032 * decision by deleting the provisions above and replace them with the notice 0033 * and other provisions required by the GPL or the LGPL. If you do not delete 0034 * the provisions above, a recipient may use your version of this file under 0035 * the terms of any one of the MPL, the GPL or the LGPL. 0036 * 0037 * ***** END LICENSE BLOCK ***** */ 0038 #ifndef nsSingleByteCharSetProber_h__ 0039 #define nsSingleByteCharSetProber_h__ 0040 0041 #include "nsCharSetProber.h" 0042 0043 #define SAMPLE_SIZE 64 0044 #define SB_ENOUGH_REL_THRESHOLD 1024 0045 #define POSITIVE_SHORTCUT_THRESHOLD (float)0.95 0046 #define NEGATIVE_SHORTCUT_THRESHOLD (float)0.05 0047 #define SYMBOL_CAT_ORDER 250 0048 #define NUMBER_OF_SEQ_CAT 4 0049 #define POSITIVE_CAT (NUMBER_OF_SEQ_CAT-1) 0050 #define NEGATIVE_CAT 0 0051 0052 typedef struct 0053 { 0054 unsigned char *charToOrderMap; // [256] table use to find a char's order 0055 char *precedenceMatrix; // [SAMPLE_SIZE][SAMPLE_SIZE]; table to find a 2-char sequence's frequency 0056 float mTypicalPositiveRatio; // = freqSeqs / totalSeqs 0057 PRBool keepEnglishLetter; // says if this script contains English characters (not implemented) 0058 const char* charsetName; 0059 } SequenceModel; 0060 0061 0062 class nsSingleByteCharSetProber : public nsCharSetProber{ 0063 public: 0064 nsSingleByteCharSetProber(SequenceModel *model) 0065 :mModel(model), mReversed(PR_FALSE), mNameProber(0) { Reset(); } 0066 nsSingleByteCharSetProber(SequenceModel *model, PRBool reversed, nsCharSetProber* nameProber) 0067 :mModel(model), mReversed(reversed), mNameProber(nameProber) { Reset(); } 0068 0069 virtual const char* GetCharSetName(); 0070 virtual nsProbingState HandleData(const char* aBuf, PRUint32 aLen); 0071 virtual nsProbingState GetState(void) {return mState;}; 0072 virtual void Reset(void); 0073 virtual float GetConfidence(void); 0074 virtual void SetOpion() {} 0075 0076 // This feature is not implemented yet. any current language model 0077 // contain this parameter as PR_FALSE. No one is looking at this 0078 // parameter or calling this method. 0079 // Moreover, the nsSBCSGroupProber which calls the HandleData of this 0080 // prober has a hard-coded call to FilterWithoutEnglishLetters which gets rid 0081 // of the English letters. 0082 PRBool KeepEnglishLetters() {return mModel->keepEnglishLetter;}; // (not implemented) 0083 0084 #ifdef DEBUG_chardet 0085 virtual void DumpStatus(); 0086 #endif 0087 0088 protected: 0089 nsProbingState mState; 0090 const SequenceModel *mModel; 0091 const PRBool mReversed; // PR_TRUE if we need to reverse every pair in the model lookup 0092 0093 //char order of last character 0094 unsigned char mLastOrder; 0095 0096 PRUint32 mTotalSeqs; 0097 PRUint32 mSeqCounters[NUMBER_OF_SEQ_CAT]; 0098 0099 PRUint32 mTotalChar; 0100 //characters that fall in our sampling range 0101 PRUint32 mFreqChar; 0102 0103 // Optional auxiliary prober for name decision. created and destroyed by the GroupProber 0104 nsCharSetProber* mNameProber; 0105 0106 }; 0107 0108 0109 extern SequenceModel Koi8rModel; 0110 extern SequenceModel Win1251Model; 0111 extern SequenceModel Latin5Model; 0112 extern SequenceModel MacCyrillicModel; 0113 extern SequenceModel Ibm866Model; 0114 extern SequenceModel Ibm855Model; 0115 extern SequenceModel Latin7Model; 0116 extern SequenceModel Win1253Model; 0117 extern SequenceModel Latin5BulgarianModel; 0118 extern SequenceModel Win1251BulgarianModel; 0119 extern SequenceModel Latin2HungarianModel; 0120 extern SequenceModel Win1250HungarianModel; 0121 extern SequenceModel Win1255Model; 0122 0123 #endif /* nsSingleByteCharSetProber_h__ */ 0124