rendered paste body class basic_decoder { protected: std::istream* is; public: basic_decoder(std::istream* _is) : is(_is) {} std::istream* istream() { return is; } virtual boost::int_fast32_t decode() = 0; virtual bool prevg() = 0; virtual bool nextg() = 0; }; // Basic Multi-Byte-Unit Decoder template<class unit_type, int byte_order> class basic_mbu_decoder : public basic_decoder { public: basic_mbu_decoder(std::istream* _is) : basic_decoder(_is) {} unit_type get_unit() { unit_type tmp; for(int i=0; i < sizeof(unit_type); ++i) { if(byte_order == BOOST_BYTE_ORDER) ((boost::uint8_t*)&tmp)[i] = is->get(); else // If the source is a different endian than the system, ((boost::uint8_t*)&tmp)[sizeof(unit_type)-i-1] = is->get(); // fill tmp backwards. } return tmp; } }; // UTF-16 Decoder template<int byte_order> class utf16_decoder : public basic_mbu_decoder<boost::uint16_t, byte_order> { public: utf16_decoder(std::istream* _is) : basic_mbu_decoder<boost::uint16_t, byte_order>(_is) {} // Read until one whole character is decoded. boost::int_fast32_t decode() { int pk = is->peek(); if(pk == EOF) return EOF; boost::uint16_t ch = get_unit(); if( istream().good() && (ch >= 0xD800 && ch <= 0xDBFF) ) // If ch is the first unit in a surrogate pair... { boost::uint16_t ch2 = get_unit(); // Get the following unit. if( istream().good() && (ch2 >= 0xDC00 && ch2 <= 0xDFFF) ) // If the following unit appropriately completes the surrogate pair... { ch = (boost::uint16_t)((ch - (boost::uint16_t)0xD800) << 10) + (ch2 - (boost::uint16_t)0xDC00) + 0x0010000; // Magic } else { throw "Unpaired surrogate."; } } return ch; } bool nextg() // Move get-pointer forward one full character. { return decode() == EOF; /* boost::uint16_t ch = get_unit(); if( istream().good() && (ch >= 0xD800 && ch <= 0xDBFF) ) { boost::uint16_t ch2 = get_unit(); if( istream().good() && (ch2 < 0xDC00 || ch2 > 0xDFFF) ) { throw "Unpaired surrogate."; } }*/ } bool prevg() // Move get-pointer back on full character. { is->seekg(std::ios::cur, -2); // Previous unit. boost::uint16_t ch = get_unit(); if(ch < 0xDC00 || ch > 0xDFFF) // If this is the second unit in a surrogate pair... { // Go back another unit to the first part of the surrogate pair. is->seekg(std::ios::cur, -2); boost::uint16_t ch = get_unit(); } return true; } }; typedef utf16_decoder<1234> utf16le_decoder; // Little endian UTF-16 Decoder typedef utf16_decoder<4321> utf16be_decoder; // Big endian UTF-16 Decoder