PHP 字符串编码处理 （附各语言的字符集编码范围）

hyytaojunming

　　PHP中GBK和UTF8编码处理 


　　
一、编码范围

　　

1. GBK (GB2312/GB18030)

x00-xff GBK双字节编码范围

x20-x7f ASCII

xa1-xff 中文

x80-xff 中文


2. UTF-8 (Unicode)

　　

u4e00-u9fa5 (中文)

x3130-x318F (韩文)

xAC00-xD7A3 (韩文)

u0800-u4e00 (日文)

ps: 韩文是大于[u9fa5]的字符


正则例子:


preg_replace("/([x80-xff])/","",$str);
preg_replace("/([u4e00-u9fa5])/","",$str);
 　　二、代码例子


//判断内容里有没有中文-GBK (PHP)
function check_is_chinese($s){
return preg_match('/[x80-xff]./', $s);
}

//获取字符串长度-GBK (PHP)
function gb_strlen($str){
$count = 0;
for($i=0; $i<strlen($str); $i++){
$s = substr($str, $i, 1);
if (preg_match("/[x80-xff]/", $s)) ++$i;
++$count;
}
return $count;
}
//截取字符串字串-GBK (PHP)
function gb_substr($str, $len){
$count = 0;
for($i=0; $i<strlen($str); $i++){
if($count == $len) break;
if(preg_match("/[x80-xff]/", substr($str, $i, 1))) ++$i;
++$count;      
}
return substr($str, 0, $i);
}

//统计字符串长度-UTF8 (PHP)
function utf8_strlen($str) {
$count = 0;
for($i = 0; $i < strlen($str); $i++){
$value = ord($str[$i]);
if($value > 127) {
$count++;
if($value >= 192 && $value <= 223) $i++;
elseif($value >= 224 && $value <= 239) $i = $i + 2;
elseif($value >= 240 && $value <= 247) $i = $i + 3;
else die('Not a UTF-8 compatible string');
}
$count++;
}
return $count;
}

//截取字符串-UTF8(PHP)
function utf8_substr($str,$position,$length){
$start_position = strlen($str);
$start_byte = 0;
$end_position = strlen($str);
$count = 0;
for($i = 0; $i < strlen($str); $i++){
if($count >= $position && $start_position > $i){
$start_position = $i;
$start_byte = $count;
}
if(($count-$start_byte)>=$length) {
$end_position = $i;
break;
}   
$value = ord($str[$i]);
if($value > 127){
$count++;
if($value >= 192 && $value <= 223) $i++;
elseif($value >= 224 && $value <= 239) $i = $i + 2;
elseif($value >= 240 && $value <= 247) $i = $i + 3;
else die('Not a UTF-8 compatible string');
}
$count++;
}
return(substr($str,$start_position,$end_position-$start_position));
}

//字符串长度统计-UTF8 [中文3个字节，俄文、韩文占2个字节，字母占1个字节] (Ruby)
def utf8_string_length(str)
temp = CGI::unescape(str)
i = 0;
j = 0;
temp.length.times{|t|
if temp[t] < 127
i += 1
elseif temp[t] >= 127 and temp[t] < 224
j += 1
if 0 == (j % 2)
i += 2
j = 0
end
else
j += 1
if 0 == (j % 3)
i +=2
j = 0
end
end
}
return i
}

//判断是否是含有韩文-UTF-8 (javascript)
function checkKoreaChar(str) {
for(i=0; i<str.length; i++) {
if(((str.charCodeAt(i) > 0x3130 && str.charCodeAt(i) < 0x318F) || (str.charCodeAt(i) >= 0xAC00 && str.charCodeAt(i) <= 0xD7A3))) {
return true;
}
}
return false;
}

//判断是否有中文字符-GBK (javascript)
function check_chinese_char(s){
return (s.length != s.replace(/[^x00-xff]/g,"**").length);
}
 　　






编码
字符集编码范围


UTF8
[\x01-\x7f]|[\xc0-\xdf][\x80-\xbf]|[\xe0-\xef][\x80-\xbf]{2}|[\xf0-\xff][\x80-\xbf]{3}


UTF16
[\x00-\xd7][\xe0-\xff]|[\xd8-\xdf][\x00-\xff]{2}


Big5
[\x01-\x7f]|[\x81-\xfe]([\x40-\x7e]|[\xa1-\xfe])


GBK
[\x01-\x7f]|[\x81-\xfe][\x40-\xfe]


GB2312汉字
[\xb0-\xf7][\xa0-\xfe]


GB2312半角标点符号及特殊符号
\xa1[\xa2-\xfe]


GB2312罗马数组及项目序号
\xa2([\xa1-\xaa]|[\xb1-\xbf]|[\xc0-\xdf]|[\xe0-\xe2]|[\xe5-\xee]|[\xf1-\xfc])


GB2312全角标点及全角字母
\xa3[\xa1-\xfe]


GB2312日文平假名
\xa4[\xa1-\xf3]


GB2312日文片假名
\xa5[\xa1-\xf6]


GB18030
[\x00-\x7f]|[\x81-\xfe][\x40-\xfe]|[\x81-\xfe][\x30-\x39][\x81-\xfe][\x30-\x39]


JIS
[\x20-\x7e]|[\x21-\x5f]|[\x21-\x7e]{2}


SJIS
[\x20-\x7e]|[\xa1-\xdf]|([\x81-\x9f]|[\xe0-\xef])([\x40-\x7e]|[\x80-\xfc])


SJIS全角空格
(?:\x81\x81)


SJIS全角数字
(?:\x82[\x4f-\x58])


SJIS全角大写英文
(?:\x82[\x60-\x79])


SJIS全角小写英文
(?:\x82[\x81-\x9a])


SJIS全角平假名
(?:\x82[\x9f-\xf1])


SJIS全角平假名扩展
(?:\x82[\x9f-\xf1]|\x81[\x4a\x4b\x54\x55])


SJIS全角片假名
(?:\x83[\x40-\x96])


SJIS全角片假名扩展
(?:\x83[\x40-\x96]|\x81[\x45\x5b\x52\x53])


EUC_JP
[\x20-\x7e]|\x81[\xa1-\xdf]|[\xa1-\xfe][\xa1-\xfe]|\x8f[\xa1-\xfe]{2}


EUC_JP标点符号及特殊字符
[\xa1-\xa2][\xa0-\xfe]


EUC_JP全角数字
\xa3[\xb0-\xb9]


EUC_JP全角大写英文
\xa3[\xc1-\xda]


EUC_JP全角小写英文
\xa3[\xe1-\xfa]


EUC_JP全角平假名
\xa4[\xa1-\xf3]


EUC_JP全角片假名
\xa3[\xb0-\xb9]|\xa3[\xc1-\xda]|\xa5[\xa1-\xf6][\xa3][\xb0-\xfa]|[\xa1][\xbc-\xbe]|[\xa1][\xdd]


EUC_JP全角汉字
[\xb0-\xcf][\xa0-\xd3]|[\xd0-\xf4][\xa0-\xfe]|[\xB0-\xF3][\xA1-\xFE]|[\xF4][\xA1-\xA6]|[\xA4][\xA1-\xF3]|[\xA5][\xA1-\xF6]|[\xA1][\xBC-\xBE]


EUC_JP全角空格
(?:\xa1\xa1)


EUC半角片假名
(?:\x8e[\xa6-\xdf])


日文半角空格
\x20

public class CnCharsetChecker {
/* Support for Chinese(GB2312) characters */
// #define isgb2312head(c) (0xa1<=(uchar)(c) && (uchar)(c)<=0xf7)
// #define isgb2312tail(c) (0xa1<=(uchar)(c) && (uchar)(c)<=0xfe)
public static boolean isGB2312(byte head, byte tail) {
int iHead = head & 0xff;
int iTail = tail & 0xff;
return ((iHead>=0xa1 && iHead<=0xf7 &&
iTail>=0xa1 && iTail<=0xfe) ? true : false);
}
/* Support for Chinese(GBK) characters */
// #define isgbkhead(c) (0x81<=(uchar)(c) && (uchar)(c)<=0xfe)
// #define isgbktail(c) ((0x40<=(uchar)(c) && (uchar)(c)<=0x7e)
//                      || (0x80<=(uchar)(c) && (uchar)(c)<=0xfe))
public static boolean isGBK(byte head, byte tail) {
int iHead = head & 0xff;
int iTail = tail & 0xff;
return ((iHead>=0x81 && iHead<=0xfe &&
(iTail>=0x40 && iTail<=0x7e ||
iTail>=0x80 && iTail<=0xfe)) ? true : false);
}
/* Support for Chinese(BIG5) characters */
// #define isbig5head(c) (0xa1<=(uchar)(c) && (uchar)(c)<=0xf9)
// #define isbig5tail(c) ((0x40<=(uchar)(c) && (uchar)(c)<=0x7e)
//                       || (0xa1<=(uchar)(c) && (uchar)(c)<=0xfe))
public static boolean isBIG5(byte head, byte tail) {
int iHead = head & 0xff;
int iTail = tail & 0xff;
return ((iHead>=0xa1 && iHead<=0xf9 &&
(iTail>=0x40 && iTail<=0x7e ||
iTail>=0xa1 && iTail<=0xfe)) ? true : false);
}
public static void main(String[] args) {
String sGB = "爱";
String sGBK = "愛";
String sBIG5 = "稲";
byte[] sChars = null;
sChars = sGB.getBytes();
System.out.println(sGB + " is " +
CnCharsetChecker.isGB2312(sChars[0],sChars[1]) + " for GB2312;" +
CnCharsetChecker.isGBK(sChars[0],sChars[1]) + " for GBK," +
CnCharsetChecker.isBIG5(sChars[0],sChars[1]) + " for BIG5");
sChars = sGBK.getBytes();
System.out.println(sGBK + " is " +
CnCharsetChecker.isGB2312(sChars[0],sChars[1]) + " for GB2312;" +
CnCharsetChecker.isGBK(sChars[0],sChars[1]) + " for GBK," +
CnCharsetChecker.isBIG5(sChars[0],sChars[1]) + " for BIG5");
sChars = sBIG5.getBytes();
System.out.println(sBIG5 + " is " +
CnCharsetChecker.isGB2312(sChars[0],sChars[1]) + " for GB2312;" +
CnCharsetChecker.isGBK(sChars[0],sChars[1]) + " for GBK," +
CnCharsetChecker.isBIG5(sChars[0],sChars[1]) + " for BIG5");
}
}