utility_module::convert_string Class Reference

Provides utilities for string encoding conversion, Base64 encoding/decoding, and substring operations like splitting or replacing. More...

#include <convert_string.h>

Collaboration diagram for utility_module::convert_string:

Public Member Functions
template<typename FromType , typename ToType >
auto	convert (const FromType &value, const std::string &from_encoding, const std::string &to_encoding) -> std::tuple< std::optional< ToType >, std::optional< std::string > >

Static Public Member Functions
static auto	to_string (const std::wstring &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a std::wstring to a std::string using the system encoding.
static auto	to_string (std::wstring_view value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a std::wstring_view to a std::string using the system encoding.
static auto	to_wstring (const std::string &value) -> std::tuple< std::optional< std::wstring >, std::optional< std::string > >
	Converts a std::string (system-encoded) to a std::wstring.
static auto	to_wstring (std::string_view value) -> std::tuple< std::optional< std::wstring >, std::optional< std::string > >
	Converts a std::string_view (system-encoded) to a std::wstring.
static auto	get_system_code_page () -> int
	Retrieves the system code page used for conversions.
static auto	system_to_utf8 (const std::string &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a system-encoded string to UTF-8.
static auto	utf8_to_system (const std::string &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a UTF-8 encoded string to the system encoding.
static auto	split (const std::string &source, const std::string &token) -> std::tuple< std::optional< std::vector< std::string > >, std::optional< std::string > >
	Splits a string by a given delimiter.
static auto	to_array (const std::string &value) -> std::tuple< std::optional< std::vector< uint8_t > >, std::optional< std::string > >
	Converts a system-encoded string to a UTF-8 byte array.
static auto	to_string (const std::vector< uint8_t > &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a UTF-8 byte array to a system-encoded string.
static auto	to_base64 (const std::vector< uint8_t > &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Encodes a byte array into a Base64 string.
static auto	from_base64 (const std::string &base64_str) -> std::tuple< std::vector< uint8_t >, std::optional< std::string > >
	Decodes a Base64 string into a byte array.
static auto	replace (std::string &source, const std::string &token, const std::string &target) -> std::optional< std::string >
	Replaces all occurrences of token in source with target, in place.
static auto	replace2 (const std::string &source, const std::string &token, const std::string &target) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Replaces all occurrences of token in source with target, returning a new string.
static auto	to_string (const std::wstring &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a std::wstring to a std::string using the system encoding.
static auto	to_string (std::wstring_view value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a std::wstring_view to a std::string using the system encoding.
static auto	to_wstring (const std::string &value) -> std::tuple< std::optional< std::wstring >, std::optional< std::string > >
	Converts a std::string (system-encoded) to a std::wstring.
static auto	to_wstring (std::string_view value) -> std::tuple< std::optional< std::wstring >, std::optional< std::string > >
	Converts a std::string_view (system-encoded) to a std::wstring.
static auto	get_system_code_page () -> int
	Retrieves the system code page used for conversions.
static auto	system_to_utf8 (const std::string &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a system-encoded string to UTF-8.
static auto	utf8_to_system (const std::string &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a UTF-8 encoded string to the system encoding.
static auto	split (const std::string &source, const std::string &token) -> std::tuple< std::optional< std::vector< std::string > >, std::optional< std::string > >
	Splits a string by a given delimiter.
static auto	to_array (const std::string &value) -> std::tuple< std::optional< std::vector< uint8_t > >, std::optional< std::string > >
	Converts a system-encoded string to a UTF-8 byte array.
static auto	to_string (const std::vector< uint8_t > &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Converts a UTF-8 byte array to a system-encoded string.
static auto	to_base64 (const std::vector< uint8_t > &value) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Encodes a byte array into a Base64 string.
static auto	from_base64 (const std::string &base64_str) -> std::tuple< std::vector< uint8_t >, std::optional< std::string > >
	Decodes a Base64 string into a byte array.
static auto	replace (std::string &source, const std::string &token, const std::string &target) -> std::optional< std::string >
	Replaces all occurrences of token in source with target, in place.
static auto	replace2 (const std::string &source, const std::string &token, const std::string &target) -> std::tuple< std::optional< std::string >, std::optional< std::string > >
	Replaces all occurrences of token in source with target, returning a new string.

Private Types
enum class	endian_types {   little , big , unknown , little ,   big , unknown }
	Possible endianness values for UTF-16 or UTF-32 data. More...
enum class	encoding_types {   utf8 , utf16 , utf32 , utf8 ,   utf16 , utf32 }
	Supported encoding types for Unicode conversion. More...
enum class	endian_types {   little , big , unknown , little ,   big , unknown }
	Possible endianness values for UTF-16 or UTF-32 data. More...
enum class	encoding_types {   utf8 , utf16 , utf32 , utf8 ,   utf16 , utf32 }
	Supported encoding types for Unicode conversion. More...

Static Private Member Functions
static auto	get_code_page_name (int code_page) -> std::string
	Retrieves a textual name for a code page (e.g., "CP_ACP" or a locale-based name).
static auto	get_encoding_name (encoding_types encoding, endian_types endian=endian_types::little) -> std::string
	Returns the encoding name string for the given encoding type and endianness.
static auto	get_wchar_encoding (endian_types endian=endian_types::little) -> std::string
	Derives the wchar_t encoding name based on its size and endianness.
template<typename FromType , typename ToType >
static auto	convert (const FromType &value, const std::string &from_encoding, const std::string &to_encoding) -> std::tuple< std::optional< ToType >, std::optional< std::string > >
	Converts from one encoding to another using simdutf.
static auto	detect_endian (const std::u16string &value) -> endian_types
	Detects the endianness of a UTF-16 string by checking for BOM or content patterns.
static auto	detect_endian (const std::u32string &value) -> endian_types
	Detects the endianness of a UTF-32 string by checking for BOM or content patterns.
static auto	has_utf8_bom (const std::string &value) -> bool
	Checks if a string has a UTF-8 BOM (Byte Order Mark).
static auto	remove_utf8_bom (const std::string &value) -> std::string
	Removes a leading UTF-8 BOM from a string, if present.
static auto	add_utf8_bom (const std::string &value) -> std::string
	Adds a UTF-8 BOM to a string if it doesn't already have one.
static auto	base64_encode (const std::vector< uint8_t > &data) -> std::string
	Encodes a byte array into a Base64 string.
static auto	base64_decode (const std::string &base64_str) -> std::tuple< std::vector< uint8_t >, std::optional< std::string > >
	Decodes a Base64 string into a byte array.
static auto	get_code_page_name (int code_page) -> std::string
	Retrieves a textual name for a code page (e.g., "CP_ACP" or a locale-based name).
static auto	get_encoding_name (encoding_types encoding, endian_types endian=endian_types::little) -> std::string
	Returns the encoding name string for the given encoding type and endianness.
static auto	get_wchar_encoding (endian_types endian=endian_types::little) -> std::string
	Derives the wchar_t encoding name based on its size and endianness.
template<typename FromType , typename ToType >
static auto	convert (const FromType &value, const std::string &from_encoding, const std::string &to_encoding) -> std::tuple< std::optional< ToType >, std::optional< std::string > >
	Converts from one encoding to another using simdutf.
static auto	detect_endian (const std::u16string &value) -> endian_types
	Detects the endianness of a UTF-16 string by checking for BOM or content patterns.
static auto	detect_endian (const std::u32string &value) -> endian_types
	Detects the endianness of a UTF-32 string by checking for BOM or content patterns.
static auto	has_utf8_bom (const std::string &value) -> bool
	Checks if a string has a UTF-8 BOM (Byte Order Mark).
static auto	remove_utf8_bom (const std::string &value) -> std::string
	Removes a leading UTF-8 BOM from a string, if present.
static auto	add_utf8_bom (const std::string &value) -> std::string
	Adds a UTF-8 BOM to a string if it doesn't already have one.
static auto	base64_encode (const std::vector< uint8_t > &data) -> std::string
	Encodes a byte array into a Base64 string.
static auto	base64_decode (const std::string &base64_str) -> std::tuple< std::vector< uint8_t >, std::optional< std::string > >
	Decodes a Base64 string into a byte array.

Detailed Description

Provides utilities for string encoding conversion, Base64 encoding/decoding, and substring operations like splitting or replacing.

This class uses simdutf for SIMD-accelerated Unicode transcoding between UTF-8, UTF-16, and UTF-32 on all platforms. It also provides methods to handle Base64 encoding/decoding and some basic string manipulation (split, replace).

Encoding Conversions

• Typically returns a tuple containing either a successful result (std::optional<T>) and an empty error (std::nullopt), or std::nullopt with an error message describing the failure.
• BOM (Byte Order Mark) is handled in certain functions for UTF-8, UTF-16, and UTF-32.

Base64

• Encodes and decodes raw byte arrays (std::vector<uint8_t>).

Usage Example

// Converting a wstring to system-encoded string
auto [strOpt, errMsg] = convert_string::to_string(L"안녕하세요");
if (!strOpt) {
    std::cerr << "Error: " << *errMsg << std::endl;
} else {
    std::cout << "Converted string: " << *strOpt << std::endl;
}

This class uses simdutf for SIMD-accelerated Unicode transcoding between UTF-8, UTF-16, and UTF-32 on all platforms. It also provides methods to handle Base64 encoding/decoding and some basic string manipulation (split, replace).

Encoding Conversions

• Typically returns a tuple containing either a successful result (std::optional<T>) and an empty error (std::nullopt), or std::nullopt with an error message describing the failure.
• BOM (Byte Order Mark) is handled in certain functions for UTF-8, UTF-16, and UTF-32.

Base64

• Encodes and decodes raw byte arrays (std::vector<uint8_t>).

Usage Example

// Converting a wstring to system-encoded string
auto [strOpt, errMsg] = convert_string::to_string(L"안녕하세요");
if (!strOpt) {
    std::cerr << "Error: " << *errMsg << std::endl;
} else {
    std::cout << "Converted string: " << *strOpt << std::endl;
}

Definition at line 52 of file convert_string.h.

Member Enumeration Documentation

encoding_types [1/2]

enum class utility_module::convert_string::encoding_types

strongprivate

Supported encoding types for Unicode conversion.

Enumerator
utf8	UTF-8 encoding.
utf16	UTF-16 encoding.
utf32	UTF-32 encoding.
utf8	UTF-8 encoding.
utf16	UTF-16 encoding.
utf32	UTF-32 encoding.

Definition at line 222 of file convert_string.h.

        {
            utf8,  
            utf16, 
            utf32  
        };

encoding_types [2/2]

enum class utility_module::convert_string::encoding_types

strongprivate

Supported encoding types for Unicode conversion.

Enumerator
utf8	UTF-8 encoding.
utf16	UTF-16 encoding.
utf32	UTF-32 encoding.
utf8	UTF-8 encoding.
utf16	UTF-16 encoding.
utf32	UTF-32 encoding.

Definition at line 216 of file convert_string.h.

        {
            utf8,  
            utf16, 
            utf32  
        };

endian_types [1/2]

enum class utility_module::convert_string::endian_types

strongprivate

Possible endianness values for UTF-16 or UTF-32 data.

Enumerator
little	Little-endian.
big	Big-endian.
unknown	Unknown or not applicable.
little	Little-endian.
big	Big-endian.
unknown	Unknown or not applicable.

Definition at line 214 of file convert_string.h.

        {
            little, 
            big,    
            unknown 
        };

endian_types [2/2]

enum class utility_module::convert_string::endian_types

strongprivate

Possible endianness values for UTF-16 or UTF-32 data.

Enumerator
little	Little-endian.
big	Big-endian.
unknown	Unknown or not applicable.
little	Little-endian.
big	Big-endian.
unknown	Unknown or not applicable.

Definition at line 208 of file convert_string.h.

        {
            little, 
            big,    
            unknown 
        };

Member Function Documentation

add_utf8_bom() [1/2]

static auto utility_module::convert_string::add_utf8_bom ( const std::string & value ) -> std::string

staticprivate

Adds a UTF-8 BOM to a string if it doesn't already have one.

Parameters

value

The input string (UTF-8 encoded).

Returns

A new string guaranteed to have the UTF-8 BOM at the start.

References base64_decode(), and base64_encode().

Here is the call graph for this function:

add_utf8_bom() [2/2]

static auto utility_module::convert_string::add_utf8_bom ( const std::string & value ) -> std::string

staticprivate

Adds a UTF-8 BOM to a string if it doesn't already have one.

Parameters

value

The input string (UTF-8 encoded).

Returns

A new string guaranteed to have the UTF-8 BOM at the start.

base64_decode() [1/2]

auto kcenon::thread::utils::convert_string::base64_decode ( const std::string & base64_str ) -> std::tuple<std::vector<uint8_t>, std::optional<std::string>>

staticprivate

Decodes a Base64 string into a byte array.

Parameters

base64_str

The Base64 string.

Returns

A tuple of:

• std::vector<uint8_t>: The decoded bytes (empty on failure).
• std::optional<std::string>: An error message if decoding fails, otherwise std::nullopt.

Definition at line 554 of file convert_string.cpp.

    {
        static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                                "abcdefghijklmnopqrstuvwxyz"
                                                "0123456789+/";
 
        if (base64_str.length() % 4 != 0)
        {
            return { std::vector<uint8_t>(), "Invalid base64 input length" };
        }
 
        size_t padding = 0;
        if (!base64_str.empty())
        {
            if (base64_str[base64_str.length() - 1] == '=')
                padding++;
            if (base64_str.length() >= 2 && base64_str[base64_str.length() - 2] == '=')
                padding++;
            if (padding > 2)
            {
                return { std::vector<uint8_t>(), "Invalid padding in base64 string" };
            }
        }
 
        size_t decoded_length = (base64_str.length() / 4) * 3 - padding;
        std::vector<uint8_t> decoded_data;
        decoded_data.reserve(decoded_length);
 
        std::vector<int> decoding_table(256, -1);
        for (int i = 0; i < 64; i++)
        {
            decoding_table[static_cast<unsigned char>(base64_chars[static_cast<size_t>(i)])] = i;
        }
 
        uint32_t buffer = 0;
        int bits_collected = 0;
        size_t i = 0;
        for (; i < base64_str.length(); ++i)
        {
            char c = base64_str[i];
            if (c == '=')
            {
                if (i < base64_str.length() - padding)
                {
                    return { std::vector<uint8_t>(), "Invalid padding position in base64 string" };
                }
                break;
            }
 
            if (decoding_table[static_cast<unsigned char>(c)] == -1)
            {
                return { std::vector<uint8_t>(), "Invalid character in base64 string" };
            }
 
            buffer = (buffer << 6) | static_cast<uint32_t>(decoding_table[static_cast<unsigned char>(c)]);
            bits_collected += 6;
 
            if (bits_collected >= 8)
            {
                bits_collected -= 8;
                decoded_data.push_back((buffer >> bits_collected) & 0xFF);
            }
        }
 
        for (; i < base64_str.length(); ++i)
        {
            if (base64_str[i] != '=')
            {
                return { std::vector<uint8_t>(),
                         "Invalid character after padding in base64 string" };
            }
        }
 
        return { decoded_data, std::nullopt };
    }

base64_decode() [2/2]

static auto utility_module::convert_string::base64_decode ( const std::string & base64_str ) -> std::tuple< std::vector< uint8_t >, std::optional< std::string > >

staticprivate

Decodes a Base64 string into a byte array.

Parameters

base64_str

The Base64 string.

Returns

A tuple of:

• std::vector<uint8_t>: The decoded bytes (empty on failure).
• std::optional<std::string>: An error message if decoding fails, otherwise std::nullopt.

Referenced by add_utf8_bom().

Here is the caller graph for this function:

base64_encode() [1/2]

auto kcenon::thread::utils::convert_string::base64_encode ( const std::vector< uint8_t > & data ) -> std::string

staticprivate

Encodes a byte array into a Base64 string.

Parameters

data	The raw byte array.

Returns

The Base64-encoded representation.

Definition at line 520 of file convert_string.cpp.

    {
        static const char base64_chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                           "abcdefghijklmnopqrstuvwxyz"
                                           "0123456789+/";
 
        std::string encoded_string;
        size_t i = 0;
        uint32_t octet_a, octet_b, octet_c;
        uint32_t triple;
 
        while (i < data.size())
        {
            octet_a = i < data.size() ? data[i++] : 0;
            octet_b = i < data.size() ? data[i++] : 0;
            octet_c = i < data.size() ? data[i++] : 0;
 
            triple = (octet_a << 16) + (octet_b << 8) + octet_c;
 
            encoded_string += base64_chars[(triple >> 18) & 0x3F];
            encoded_string += base64_chars[(triple >> 12) & 0x3F];
            encoded_string += base64_chars[(triple >> 6) & 0x3F];
            encoded_string += base64_chars[triple & 0x3F];
        }
 
        int mod_table[] = { 0, 2, 1 };
        for (int j = 0; j < mod_table[data.size() % 3]; j++)
        {
            encoded_string[encoded_string.size() - 1 - static_cast<size_t>(j)] = '=';
        }
 
        return encoded_string;
    }

base64_encode() [2/2]

static auto utility_module::convert_string::base64_encode ( const std::vector< uint8_t > & data ) -> std::string

staticprivate

Encodes a byte array into a Base64 string.

Parameters

data	The raw byte array.

Returns

The Base64-encoded representation.

Referenced by add_utf8_bom().

Here is the caller graph for this function:

convert() [1/3]

template<typename FromType , typename ToType >

static auto utility_module::convert_string::convert ( const FromType & value, const std::string & from_encoding, const std::string & to_encoding ) -> std::tuple< std::optional< ToType >, std::optional< std::string > >

staticprivate

Converts from one encoding to another using simdutf.

Template Parameters

FromType	The input string type.
ToType	The output string type.

Parameters

value	The input string.
from_encoding	Source encoding name (e.g., "UTF-16LE").
to_encoding	Target encoding name (e.g., "UTF-8").

Returns

A tuple:

• std::optional<ToType>: The converted string on success, or std::nullopt on failure.
• std::optional<std::string>: The error message if conversion fails.

convert() [2/3]

template<typename FromType , typename ToType >

auto utility_module::convert_string::convert	(	const FromType &	value,
		const std::string &	from_encoding,
		const std::string &	to_encoding ) -> std::tuple<std::optional<ToType>, std::optional<std::string>>

Definition at line 36 of file convert_string.cpp.

    {
        if (value.empty())
        {
            return { ToType{}, std::nullopt };
        }
 
        if constexpr (std::is_same_v<FromType, ToType>)
        {
            if (from_encoding == to_encoding)
            {
                return { ToType(value), std::nullopt };
            }
        }
 
        const auto* src_data = value.data();
        size_t src_len = value.size();
 
        // UTF-8 -> UTF-16LE (std::string -> std::wstring on Windows, or char16_t-based)
        if (from_encoding == "UTF-8"
            && (to_encoding == "UTF-16LE" || to_encoding == "UTF-16"))
        {
            if (!simdutf::validate_utf8(reinterpret_cast<const char*>(src_data),
                                        src_len * sizeof(typename FromType::value_type)))
            {
                return { std::nullopt, "Invalid UTF-8 input" };
            }
 
            size_t utf16_len = simdutf::utf16_length_from_utf8(
                reinterpret_cast<const char*>(src_data),
                src_len * sizeof(typename FromType::value_type));
 
            if constexpr (sizeof(typename ToType::value_type) == 2)
            {
                ToType result(utf16_len, typename ToType::value_type{});
                size_t written = simdutf::convert_utf8_to_utf16le(
                    reinterpret_cast<const char*>(src_data),
                    src_len * sizeof(typename FromType::value_type),
                    reinterpret_cast<char16_t*>(result.data()));
                if (written == 0 && utf16_len > 0)
                {
                    return { std::nullopt, "UTF-8 to UTF-16LE conversion failed" };
                }
                result.resize(written);
                return { result, std::nullopt };
            }
        }
 
        // UTF-8 -> UTF-32LE (std::string -> std::wstring on Unix)
        if (from_encoding == "UTF-8"
            && (to_encoding == "UTF-32LE" || to_encoding == "UTF-32"))
        {
            if (!simdutf::validate_utf8(reinterpret_cast<const char*>(src_data),
                                        src_len * sizeof(typename FromType::value_type)))
            {
                return { std::nullopt, "Invalid UTF-8 input" };
            }
 
            size_t utf32_len = simdutf::utf32_length_from_utf8(
                reinterpret_cast<const char*>(src_data),
                src_len * sizeof(typename FromType::value_type));
 
            if constexpr (sizeof(typename ToType::value_type) == 4)
            {
                ToType result(utf32_len, typename ToType::value_type{});
                size_t written = simdutf::convert_utf8_to_utf32(
                    reinterpret_cast<const char*>(src_data),
                    src_len * sizeof(typename FromType::value_type),
                    reinterpret_cast<char32_t*>(result.data()));
                if (written == 0 && utf32_len > 0)
                {
                    return { std::nullopt, "UTF-8 to UTF-32 conversion failed" };
                }
                result.resize(written);
                return { result, std::nullopt };
            }
        }
 
        // UTF-16LE -> UTF-8 (std::wstring -> std::string on Windows)
        if ((from_encoding == "UTF-16LE" || from_encoding == "UTF-16")
            && to_encoding == "UTF-8")
        {
            if constexpr (sizeof(typename FromType::value_type) == 2)
            {
                const char16_t* utf16_data
                    = reinterpret_cast<const char16_t*>(src_data);
 
                if (!simdutf::validate_utf16le(utf16_data, src_len))
                {
                    return { std::nullopt, "Invalid UTF-16LE input" };
                }
 
                size_t utf8_len
                    = simdutf::utf8_length_from_utf16le(utf16_data, src_len);
 
                if constexpr (std::is_same_v<ToType, std::string>)
                {
                    std::string result(utf8_len, '\0');
                    size_t written = simdutf::convert_utf16le_to_utf8(
                        utf16_data, src_len, result.data());
                    if (written == 0 && utf8_len > 0)
                    {
                        return { std::nullopt,
                                 "UTF-16LE to UTF-8 conversion failed" };
                    }
                    result.resize(written);
                    return { result, std::nullopt };
                }
            }
        }
 
        // UTF-32LE -> UTF-8 (std::wstring -> std::string on Unix)
        if ((from_encoding == "UTF-32LE" || from_encoding == "UTF-32")
            && to_encoding == "UTF-8")
        {
            if constexpr (sizeof(typename FromType::value_type) == 4)
            {
                const char32_t* utf32_data
                    = reinterpret_cast<const char32_t*>(src_data);
 
                if (!simdutf::validate_utf32(utf32_data, src_len))
                {
                    return { std::nullopt, "Invalid UTF-32 input" };
                }
 
                size_t utf8_len
                    = simdutf::utf8_length_from_utf32(utf32_data, src_len);
 
                if constexpr (std::is_same_v<ToType, std::string>)
                {
                    std::string result(utf8_len, '\0');
                    size_t written = simdutf::convert_utf32_to_utf8(
                        utf32_data, src_len, result.data());
                    if (written == 0 && utf8_len > 0)
                    {
                        return { std::nullopt,
                                 "UTF-32 to UTF-8 conversion failed" };
                    }
                    result.resize(written);
                    return { result, std::nullopt };
                }
            }
        }
 
        return { std::nullopt,
                 std::format("Unsupported encoding conversion: {} -> {}",
                             from_encoding, to_encoding) };
    }

convert() [3/3]

template<typename FromType , typename ToType >

static auto utility_module::convert_string::convert ( const FromType & value, const std::string & from_encoding, const std::string & to_encoding ) -> std::tuple< std::optional< ToType >, std::optional< std::string > >

staticprivate

Converts from one encoding to another using simdutf.

Template Parameters

FromType	The input string type.
ToType	The output string type.

Parameters

value	The input string.
from_encoding	Source encoding name (e.g., "UTF-16LE").
to_encoding	Target encoding name (e.g., "UTF-8").

Returns

A tuple:

• std::optional<ToType>: The converted string on success, or std::nullopt on failure.
• std::optional<std::string>: The error message if conversion fails.

detect_endian() [1/4]

static auto utility_module::convert_string::detect_endian ( const std::u16string & value ) -> endian_types

staticprivate

Detects the endianness of a UTF-16 string by checking for BOM or content patterns.

Parameters

value

The UTF-16 string.

Returns

The detected endianness (little, big, or unknown).

detect_endian() [2/4]

static auto utility_module::convert_string::detect_endian ( const std::u16string & value ) -> endian_types

staticprivate

Detects the endianness of a UTF-16 string by checking for BOM or content patterns.

Parameters

value

The UTF-16 string.

Returns

The detected endianness (little, big, or unknown).

detect_endian() [3/4]

static auto utility_module::convert_string::detect_endian ( const std::u32string & value ) -> endian_types

staticprivate

Detects the endianness of a UTF-32 string by checking for BOM or content patterns.

Parameters

value

The UTF-32 string.

Returns

The detected endianness (little, big, or unknown).

detect_endian() [4/4]

static auto utility_module::convert_string::detect_endian ( const std::u32string & value ) -> endian_types

staticprivate

Detects the endianness of a UTF-32 string by checking for BOM or content patterns.

Parameters

value

The UTF-32 string.

Returns

The detected endianness (little, big, or unknown).

from_base64() [1/2]

auto kcenon::thread::utils::convert_string::from_base64 ( const std::string & base64_str ) -> std::tuple<std::vector<uint8_t>, std::optional<std::string>>

static

Decodes a Base64 string into a byte array.

Parameters

base64_str

The Base64-encoded string.

Returns

A tuple of:

• std::vector<uint8_t>: The decoded data. May be empty if decoding fails.
• std::optional<std::string>: An error message on failure, or std::nullopt on success.

Definition at line 470 of file convert_string.cpp.

    {
        return base64_decode(base64_str);
    }

from_base64() [2/2]

static auto utility_module::convert_string::from_base64 ( const std::string & base64_str ) -> std::tuple< std::vector< uint8_t >, std::optional< std::string > >

static

Decodes a Base64 string into a byte array.

Parameters

base64_str

The Base64-encoded string.

Returns

A tuple of:

• std::vector<uint8_t>: The decoded data. May be empty if decoding fails.
• std::optional<std::string>: An error message on failure, or std::nullopt on success.

get_code_page_name() [1/2]

auto kcenon::thread::utils::convert_string::get_code_page_name ( int code_page ) -> std::string

staticprivate

Retrieves a textual name for a code page (e.g., "CP_ACP" or a locale-based name).

Parameters

code_page

The code page integer.

Returns

A string representing the code page name, or an empty string if unknown.

Definition at line 374 of file convert_string.cpp.

    {
        switch (code_page)
        {
        case 65001:
            return "UTF-8";
        default:
            return "CP" + std::to_string(code_page);
        }
    }

get_code_page_name() [2/2]

static auto utility_module::convert_string::get_code_page_name ( int code_page ) -> std::string

staticprivate

Retrieves a textual name for a code page (e.g., "CP_ACP" or a locale-based name).

Parameters

code_page

The code page integer.

Returns

A string representing the code page name, or an empty string if unknown.

get_encoding_name() [1/2]

auto kcenon::thread::utils::convert_string::get_encoding_name ( encoding_types encoding, endian_types endian = endian_types::little ) -> std::string

staticprivate

Returns the encoding name string for the given encoding type and endianness.

Parameters

encoding	The base encoding (UTF-8, UTF-16, or UTF-32).
endian	The byte order (little, big) if applicable.

Returns

An encoding name string such as "UTF-8" or "UTF-16LE".

Definition at line 248 of file convert_string.cpp.

    {
        switch (encoding)
        {
        case encoding_types::utf8:
            return "UTF-8";
        case encoding_types::utf16:
            if (endian == endian_types::little)
                return "UTF-16LE";
            else if (endian == endian_types::big)
                return "UTF-16BE";
            else
                return "UTF-16";
        case encoding_types::utf32:
            if (endian == endian_types::little)
                return "UTF-32LE";
            else if (endian == endian_types::big)
                return "UTF-32BE";
            else
                return "UTF-32";
        default:
            throw std::runtime_error("Unknown encoding");
        }
    }

get_encoding_name() [2/2]

static auto utility_module::convert_string::get_encoding_name ( encoding_types encoding, endian_types endian = endian_types::little ) -> std::string

staticprivate

Returns the encoding name string for the given encoding type and endianness.

Parameters

encoding	The base encoding (UTF-8, UTF-16, or UTF-32).
endian	The byte order (little, big) if applicable.

Returns

An encoding name string such as "UTF-8" or "UTF-16LE".

get_system_code_page() [1/2]

auto kcenon::thread::utils::convert_string::get_system_code_page ( ) -> int

static

Retrieves the system code page used for conversions.

Returns

An integer representing the system code page (e.g., CP_ACP on Windows).

On Unix-like systems, returns UTF-8 (65001) as the default. On Windows, uses GetACP() to detect the system code page.

Definition at line 365 of file convert_string.cpp.

    {
#ifdef _WIN32
        return static_cast<int>(GetACP());
#else
        return 65001;
#endif
    }

get_system_code_page() [2/2]

static auto utility_module::convert_string::get_system_code_page ( ) -> int

static

Retrieves the system code page used for conversions.

Returns

An integer representing the system code page (e.g., CP_ACP on Windows).

On Unix-like systems, returns UTF-8 (65001) as the default. On Windows, uses GetACP() to detect the system code page.

get_wchar_encoding() [1/2]

auto kcenon::thread::utils::convert_string::get_wchar_encoding ( endian_types endian = endian_types::little ) -> std::string

staticprivate

Derives the wchar_t encoding name based on its size and endianness.

Parameters

endian

The byte order. Defaults to little-endian.

Returns

For example, "UTF-16LE" on Windows or "UTF-32LE" on some Unix-like systems.

Definition at line 274 of file convert_string.cpp.

    {
        if constexpr (sizeof(wchar_t) == 2)
        {
            return get_encoding_name(encoding_types::utf16, endian);
        }
        else if constexpr (sizeof(wchar_t) == 4)
        {
            return get_encoding_name(encoding_types::utf32, endian);
        }
        else
        {
            throw std::runtime_error("Unsupported wchar_t size");
        }
    }

get_wchar_encoding() [2/2]

static auto utility_module::convert_string::get_wchar_encoding ( endian_types endian = endian_types::little ) -> std::string

staticprivate

Derives the wchar_t encoding name based on its size and endianness.

Parameters

endian

The byte order. Defaults to little-endian.

Returns

For example, "UTF-16LE" on Windows or "UTF-32LE" on some Unix-like systems.

has_utf8_bom() [1/2]

static auto utility_module::convert_string::has_utf8_bom ( const std::string & value ) -> bool

staticprivate

Checks if a string has a UTF-8 BOM (Byte Order Mark).

Parameters

value

The input string (UTF-8 encoded).

Returns

true if the string starts with the UTF-8 BOM (0xEF,0xBB,0xBF), else false.

has_utf8_bom() [2/2]

static auto utility_module::convert_string::has_utf8_bom ( const std::string & value ) -> bool

staticprivate

Checks if a string has a UTF-8 BOM (Byte Order Mark).

Parameters

value

The input string (UTF-8 encoded).

Returns

true if the string starts with the UTF-8 BOM (0xEF,0xBB,0xBF), else false.

remove_utf8_bom() [1/2]

static auto utility_module::convert_string::remove_utf8_bom ( const std::string & value ) -> std::string

staticprivate

Removes a leading UTF-8 BOM from a string, if present.

Parameters

value

The input string (UTF-8 encoded).

Returns

A new string without the BOM. If no BOM was present, returns the original string.

remove_utf8_bom() [2/2]

static auto utility_module::convert_string::remove_utf8_bom ( const std::string & value ) -> std::string

staticprivate

Removes a leading UTF-8 BOM from a string, if present.

Parameters

value

The input string (UTF-8 encoded).

Returns

A new string without the BOM. If no BOM was present, returns the original string.

replace() [1/2]

auto kcenon::thread::utils::convert_string::replace ( std::string & source, const std::string & token, const std::string & target ) -> std::optional<std::string>

static

Replaces all occurrences of token in source with target, in place.

Parameters

source	The string to modify.
token	The substring to find.
target	The substring to replace token with.

Returns

std::optional<std::string> containing an error message if replacement fails, or std::nullopt on success.

This function modifies source directly. If token is empty, it returns an error.

Definition at line 476 of file convert_string.cpp.

    {
        auto [value, value_error] = replace2(source, token, target);
        if (value_error.has_value())
        {
            return value_error;
        }
 
        source = value.value();
        return std::nullopt;
    }

replace() [2/2]

static auto utility_module::convert_string::replace ( std::string & source, const std::string & token, const std::string & target ) -> std::optional< std::string >

static

Replaces all occurrences of token in source with target, in place.

Parameters

source	The string to modify.
token	The substring to find.
target	The substring to replace token with.

Returns

std::optional<std::string> containing an error message if replacement fails, or std::nullopt on success.

This function modifies source directly. If token is empty, it returns an error.

replace2() [1/2]

auto kcenon::thread::utils::convert_string::replace2 ( const std::string & source, const std::string & token, const std::string & target ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Replaces all occurrences of token in source with target, returning a new string.

Parameters

source	The source string (unchanged).
token	The substring to find.
target	The substring to replace token with.

Returns

A tuple of:

• std::optional<std::string>: The modified string on success.
• std::optional<std::string>: An error message if replacement fails.

Definition at line 490 of file convert_string.cpp.

    {
        if (source.empty())
        {
            return { std::nullopt, "Source string is empty" };
        }
 
        if (token.empty())
        {
            return { std::nullopt, "Token string is empty" };
        }
 
        std::string result;
 
        size_t last_offset = 0;
        for (size_t offset = source.find(token, last_offset); offset != std::string::npos;
             last_offset = offset + token.size(), offset = source.find(token, last_offset))
        {
            std::format_to(std::back_inserter(result), "{}{}",
                                 source.substr(last_offset, offset - last_offset), target);
        }
 
        std::format_to(std::back_inserter(result), "{}", source.substr(last_offset));
 
        return { result, std::nullopt };
    }

replace2() [2/2]

static auto utility_module::convert_string::replace2 ( const std::string & source, const std::string & token, const std::string & target ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Replaces all occurrences of token in source with target, returning a new string.

Parameters

source	The source string (unchanged).
token	The substring to find.
target	The substring to replace token with.

Returns

A tuple of:

• std::optional<std::string>: The modified string on success.
• std::optional<std::string>: An error message if replacement fails.

split() [1/2]

auto kcenon::thread::utils::convert_string::split ( const std::string & source, const std::string & token ) -> std::tuple<std::optional<std::vector<std::string>>, std::optional<std::string>>

static

Splits a string by a given delimiter.

Parameters

source	The source string to split.
token	The delimiter substring.

Returns

A tuple of:

• std::optional<std::vector<std::string>>: A vector of tokens on success.
• std::optional<std::string>: An error message if splitting fails.

If token is empty, splitting cannot proceed and an error is returned.

Definition at line 409 of file convert_string.cpp.

    {
        if (token.empty())
        {
            return { std::vector{ source }, std::nullopt };
        }
 
        std::vector<std::string> result;
        size_t start_pos = 0;
        size_t end_pos = source.find(token);
 
        while (end_pos != std::string::npos)
        {
            result.emplace_back(source.substr(start_pos, end_pos - start_pos));
            start_pos = end_pos + token.length();
            end_pos = source.find(token, start_pos);
        }
 
        result.emplace_back(source.substr(start_pos));
 
        return { result, std::nullopt };
    }

split() [2/2]

static auto utility_module::convert_string::split ( const std::string & source, const std::string & token ) -> std::tuple< std::optional< std::vector< std::string > >, std::optional< std::string > >

static

Splits a string by a given delimiter.

Parameters

source	The source string to split.
token	The delimiter substring.

Returns

A tuple of:

• std::optional<std::vector<std::string>>: A vector of tokens on success.
• std::optional<std::string>: An error message if splitting fails.

If token is empty, splitting cannot proceed and an error is returned.

system_to_utf8() [1/2]

auto kcenon::thread::utils::convert_string::system_to_utf8 ( const std::string & value ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Converts a system-encoded string to UTF-8.

Parameters

value

The input string in system encoding.

Returns

A tuple of:

• std::optional<std::string>: The UTF-8 encoded string on success.
• std::optional<std::string>: An error message on failure.

Definition at line 385 of file convert_string.cpp.

    {
        int code_page = get_system_code_page();
        if (code_page == 65001)
        {
            return { value, std::nullopt };
        }
        return convert<std::string, std::string>(value, get_code_page_name(code_page),
                                                 get_encoding_name(encoding_types::utf8));
    }

system_to_utf8() [2/2]

static auto utility_module::convert_string::system_to_utf8 ( const std::string & value ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Converts a system-encoded string to UTF-8.

Parameters

value

The input string in system encoding.

Returns

A tuple of:

• std::optional<std::string>: The UTF-8 encoded string on success.
• std::optional<std::string>: An error message on failure.

to_array() [1/2]

auto kcenon::thread::utils::convert_string::to_array ( const std::string & value ) -> std::tuple<std::optional<std::vector<uint8_t>>, std::optional<std::string>>

static

Converts a system-encoded string to a UTF-8 byte array.

Parameters

value

The input string in system encoding.

Returns

A tuple of:

• std::optional<std::vector<uint8_t>>: The UTF-8 byte array on success.
• std::optional<std::string>: An error message on failure.

Definition at line 433 of file convert_string.cpp.

    {
        auto [utf8, convert_error] = system_to_utf8(value);
        if (convert_error.has_value())
        {
            return { std::nullopt, convert_error };
        }
 
        auto utf8_no_bom = remove_utf8_bom(utf8.value());
 
        return { std::vector<uint8_t>(utf8_no_bom.begin(), utf8_no_bom.end()), std::nullopt };
    }

to_array() [2/2]

static auto utility_module::convert_string::to_array ( const std::string & value ) -> std::tuple< std::optional< std::vector< uint8_t > >, std::optional< std::string > >

static

Converts a system-encoded string to a UTF-8 byte array.

Parameters

value

The input string in system encoding.

Returns

A tuple of:

• std::optional<std::vector<uint8_t>>: The UTF-8 byte array on success.
• std::optional<std::string>: An error message on failure.

to_base64() [1/2]

auto kcenon::thread::utils::convert_string::to_base64 ( const std::vector< uint8_t > & value ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Encodes a byte array into a Base64 string.

Parameters

value

The raw byte array to encode.

Returns

A tuple of:

• std::optional<std::string>: The Base64-encoded string on success.
• std::optional<std::string>: An error message if encoding fails.

Typically, Base64 encoding should not fail unless the input is extremely large and memory allocation fails.

Definition at line 456 of file convert_string.cpp.

    {
        try
        {
            std::string encoded = base64_encode(value);
            return { encoded, std::nullopt };
        }
        catch (const std::exception& e)
        {
            return { std::nullopt, e.what() };
        }
    }

to_base64() [2/2]

static auto utility_module::convert_string::to_base64 ( const std::vector< uint8_t > & value ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Encodes a byte array into a Base64 string.

Parameters

value

The raw byte array to encode.

Returns

A tuple of:

• std::optional<std::string>: The Base64-encoded string on success.
• std::optional<std::string>: An error message if encoding fails.

Typically, Base64 encoding should not fail unless the input is extremely large and memory allocation fails.

to_string() [1/6]

auto kcenon::thread::utils::convert_string::to_string ( const std::vector< uint8_t > & value ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Converts a UTF-8 byte array to a system-encoded string.

Parameters

value

The input byte array.

Returns

A tuple with:

• std::optional<std::string>: The system-encoded string on success.
• std::optional<std::string>: An error message on failure.

Definition at line 447 of file convert_string.cpp.

    {
        std::string utf8(value.begin(), value.end());
        auto utf8_no_bom = remove_utf8_bom(utf8);
 
        return utf8_to_system(utf8);
    }

to_string() [2/6]

static auto utility_module::convert_string::to_string ( const std::vector< uint8_t > & value ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Converts a UTF-8 byte array to a system-encoded string.

Parameters

value

The input byte array.

Returns

A tuple with:

• std::optional<std::string>: The system-encoded string on success.
• std::optional<std::string>: An error message on failure.

to_string() [3/6]

auto kcenon::thread::utils::convert_string::to_string ( const std::wstring & value ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Converts a std::wstring to a std::string using the system encoding.

Parameters

value

The wide-string input.

Returns

A tuple of:

• std::optional<std::string>: The converted narrow string on success, or std::nullopt on failure.
• std::optional<std::string>: The error message if conversion fails, otherwise std::nullopt.

Uses simdutf for SIMD-accelerated UTF-8/UTF-16/UTF-32 transcoding on all platforms.

Definition at line 188 of file convert_string.cpp.

    {
        auto [result, error]
            = convert<std::wstring, std::string>(value, get_wchar_encoding(endian_types::little),
                                                 get_encoding_name(encoding_types::utf8));
 
        if (result.has_value())
        {
            return utf8_to_system(result.value());
        }
 
        return { std::nullopt, error };
    }

to_string() [4/6]

static auto utility_module::convert_string::to_string ( const std::wstring & value ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Converts a std::wstring to a std::string using the system encoding.

Parameters

value

The wide-string input.

Returns

A tuple of:

• std::optional<std::string>: The converted narrow string on success, or std::nullopt on failure.
• std::optional<std::string>: The error message if conversion fails, otherwise std::nullopt.

Uses simdutf for SIMD-accelerated UTF-8/UTF-16/UTF-32 transcoding on all platforms.

to_string() [5/6]

auto kcenon::thread::utils::convert_string::to_string ( std::wstring_view value ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Converts a std::wstring_view to a std::string using the system encoding.

Parameters

value

The wide-string view.

Returns

Same tuple semantics as to_string(const std::wstring&).

This is a convenience overload for handling string views without copying the entire wide string first.

Definition at line 203 of file convert_string.cpp.

    {
        auto [result, error] = convert<std::wstring_view, std::string>(
            value, get_wchar_encoding(endian_types::little),
            get_encoding_name(encoding_types::utf8));
 
        if (result.has_value())
        {
            return utf8_to_system(result.value());
        }
 
        return { std::nullopt, error };
    }

to_string() [6/6]

static auto utility_module::convert_string::to_string ( std::wstring_view value ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Converts a std::wstring_view to a std::string using the system encoding.

Parameters

value

The wide-string view.

Returns

Same tuple semantics as to_string(const std::wstring&).

This is a convenience overload for handling string views without copying the entire wide string first.

to_wstring() [1/4]

auto kcenon::thread::utils::convert_string::to_wstring ( const std::string & value ) -> std::tuple<std::optional<std::wstring>, std::optional<std::string>>

static

Converts a std::string (system-encoded) to a std::wstring.

Parameters

value

The narrow string in system encoding.

Returns

A tuple containing:

• std::optional<std::wstring>: The converted wide string on success.
• std::optional<std::string>: An error message if conversion fails.

Definition at line 218 of file convert_string.cpp.

    {
        auto [result, err] = system_to_utf8(value);
        if (!result.has_value())
        {
            return { std::nullopt, err };
        }
 
        std::string clean_value = remove_utf8_bom(result.value());
        endian_types endian = endian_types::little;
        return convert<std::string, std::wstring>(
            clean_value, get_encoding_name(encoding_types::utf8), get_wchar_encoding(endian));
    }

to_wstring() [2/4]

static auto utility_module::convert_string::to_wstring ( const std::string & value ) -> std::tuple< std::optional< std::wstring >, std::optional< std::string > >

static

Converts a std::string (system-encoded) to a std::wstring.

Parameters

value

The narrow string in system encoding.

Returns

A tuple containing:

• std::optional<std::wstring>: The converted wide string on success.
• std::optional<std::string>: An error message if conversion fails.

Referenced by std::formatter< kcenon::thread::job_types, wchar_t >::format(), std::formatter< kcenon::thread::thread_conditions, wchar_t >::format(), std::formatter< kcenon::thread::thread_pool, wchar_t >::format(), std::formatter< kcenon::thread::thread_worker, wchar_t >::format(), std::formatter< kcenon::thread::typed_thread_pool_t< job_type >, wchar_t >::format(), and std::formatter< kcenon::thread::typed_thread_worker_t< job_type >, wchar_t >::format().

Here is the caller graph for this function:

to_wstring() [3/4]

auto kcenon::thread::utils::convert_string::to_wstring ( std::string_view value ) -> std::tuple<std::optional<std::wstring>, std::optional<std::string>>

static

Converts a std::string_view (system-encoded) to a std::wstring.

Parameters

value

The narrow string view in system encoding.

Returns

Same tuple semantics as to_wstring(const std::string&).

Definition at line 233 of file convert_string.cpp.

    {
        auto [result, err] = system_to_utf8(std::string(value));
        if (!result.has_value())
        {
            return { std::nullopt, err };
        }
 
        std::string clean_value = remove_utf8_bom(result.value());
        endian_types endian = endian_types::little;
        return convert<std::string, std::wstring>(
            clean_value, get_encoding_name(encoding_types::utf8), get_wchar_encoding(endian));
    }

to_wstring() [4/4]

static auto utility_module::convert_string::to_wstring ( std::string_view value ) -> std::tuple< std::optional< std::wstring >, std::optional< std::string > >

static

Converts a std::string_view (system-encoded) to a std::wstring.

Parameters

value

The narrow string view in system encoding.

Returns

Same tuple semantics as to_wstring(const std::string&).

utf8_to_system() [1/2]

auto kcenon::thread::utils::convert_string::utf8_to_system ( const std::string & value ) -> std::tuple<std::optional<std::string>, std::optional<std::string>>

static

Converts a UTF-8 encoded string to the system encoding.

Parameters

value

The UTF-8 encoded input string.

Returns

Same tuple semantics as system_to_utf8().

Definition at line 397 of file convert_string.cpp.

    {
        int code_page = get_system_code_page();
        if (code_page == 65001)
        {
            return { value, std::nullopt };
        }
        return convert<std::string, std::string>(value, get_encoding_name(encoding_types::utf8),
                                                 get_code_page_name(code_page));
    }

utf8_to_system() [2/2]

static auto utility_module::convert_string::utf8_to_system ( const std::string & value ) -> std::tuple< std::optional< std::string >, std::optional< std::string > >

static

Converts a UTF-8 encoded string to the system encoding.

Parameters

value

The UTF-8 encoded input string.

Returns

Same tuple semantics as system_to_utf8().

---

The documentation for this class was generated from the following files:

• utilities/include/convert_string.h
• include/kcenon/thread/utils/convert_string.h
• src/utils/convert_string.cpp