Mirror
/
Tengine
mirror of https://github.com/alibaba/tengine.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

iwrr: a wrr with always O(1) time and O(n) memory (#1729)

pull/1898/head
纪卓志 2 years ago
committed by GitHub
parent
cd2b982068
commit
8a7529d074
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 838 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      .github/workflows/ci-arm64.yml
  2. 1
      .github/workflows/ci.yml
  3. 61
      docs/modules/ngx_http_upstream_iwrr_module.md
  4. 62
      docs/modules/ngx_http_upstream_iwrr_module_cn.md
  5. 14
      modules/ngx_http_upstream_iwrr_module/config
  6. 465
      modules/ngx_http_upstream_iwrr_module/ngx_http_upstream_iwrr_module.c
  7. 234
      tests/nginx-tests/tengine-tests/iwrr.t

1
.github/workflows/ci-arm64.yml
View File

@ -77,6 +77,7 @@ jobs:
--add-module=./modules/ngx_http_upstream_consistent_hash_module \
--add-module=./modules/ngx_http_upstream_dynamic_module \
--add-module=./modules/ngx_http_upstream_dyups_module \
--add-module=./modules/ngx_http_upstream_iwrr_module \
--add-module=./modules/ngx_http_upstream_keepalive_module \
--add-module=./modules/ngx_http_upstream_session_sticky_module \
--add-module=./modules/ngx_http_upstream_vnswrr_module \

1
.github/workflows/ci.yml
View File

@ -91,6 +91,7 @@ jobs:
--add-module=./modules/ngx_http_upstream_consistent_hash_module \
--add-module=./modules/ngx_http_upstream_dynamic_module \
--add-module=./modules/ngx_http_upstream_dyups_module \
--add-module=./modules/ngx_http_upstream_iwrr_module \
--add-module=./modules/ngx_http_upstream_keepalive_module \
--add-module=./modules/ngx_http_upstream_session_sticky_module \
--add-module=./modules/ngx_http_upstream_vnswrr_module \

61
docs/modules/ngx_http_upstream_iwrr_module.md
View File

@ -0,0 +1,61 @@
## Name
ngx_http_upstream_iwrr_module.
## Introduction
The `IWRR` module is an efficient load balancing algorithm with `O(1)` time complexity, but `IWRR` is no need to incremental initialization.
Compared with Nginx's official `SWRR` algorithm and `VNSWRR`, `IWRR` abandons smoothness on the premise of ensuring the correctness of the weighted load balancing algorithm, ensuring that no matter how the total weight of the cluster changes, `IWRR` space The complexity is always `O(n)`.
## Example
```
http {
upstream backend {
iwrr; # enable IWRR load balancing algorithm.
127.0.0.1 port=81;
127.0.0.1 port=82 weight=2;
127.0.0.1 port=83;
127.0.0.1 port=84 backup;
127.0.0.1 port=85 down;
}
server {
server_name localhost;
location / {
proxy_pass http://backend;
}
}
}
```
## Installation
Build Tengine with this module from source:
```
./configure --add-module=./modules/ngx_http_upstream_iwrr_module/
make
make install
```
## Directive
iwrr
=======
```
Syntax: iwrr [max_init=number]
Default: none
Context: upstream
```
Enable `iwrr` load balancing algorithm.

62
docs/modules/ngx_http_upstream_iwrr_module_cn.md
View File

@ -0,0 +1,62 @@
## 名称
ngx_http_upstream_iwrr_module.
## 介绍
`IWRR`模块是一个高效的负载均衡算法，与`VNSWRR`相同，它具有`O(1)`的时间复杂度，但是`IWRR`不需要执行渐进式初始化操作。
同Nginx官方的加权轮询负载均衡算法及`VNSWRR`相比，`IWRR`在保证加权负载均衡算法正确性的前提下，牺牲了平滑的特点，保证无论集群总权重如何变化，`IWRR`空间复杂度总是`O(n)`的。
## 配置列子
```
http {
upstream backend {
iwrr; # enable IWRR load balancing algorithm.
127.0.0.1 port=81;
127.0.0.1 port=82 weight=2;
127.0.0.1 port=83;
127.0.0.1 port=84 backup;
127.0.0.1 port=85 down;
}
server {
server_name localhost;
location / {
proxy_pass http://backend;
}
}
}
```
## 安装方法
在Tengine中，通过源码安装此模块：
```
./configure --add-module=./modules/ngx_http_upstream_iwrr_module
make
make install
```
## 指令描述
iwrr
=======
```
Syntax: iwrr
Default: none
Context: upstream
```
在upstream里面启用 `iwrr` 加权轮询算法。

14
modules/ngx_http_upstream_iwrr_module/config
View File

@ -0,0 +1,14 @@
ngx_addon_name=ngx_http_upstream_iwrr_module
HTTP_UPSTREAM_IWRR_SRCS="$ngx_addon_dir/ngx_http_upstream_iwrr_module.c"
if test -n "$ngx_module_link"; then
ngx_module_type=HTTP
ngx_module_name=$ngx_addon_name
ngx_module_deps=
ngx_module_srcs="$HTTP_UPSTREAM_IWRR_SRCS"
. auto/module
else
HTTP_MODULES="$HTTP_MODULES ngx_http_upstream_iwrr_module"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS $HTTP_UPSTREAM_IWRR_SRCS"
fi

465
modules/ngx_http_upstream_iwrr_module/ngx_http_upstream_iwrr_module.c
View File

@ -0,0 +1,465 @@
/*
* Copyright (C) 2010-2023 Alibaba Group Holding Limited
* Copyright (C) 2010-2023 Zhuozhi Ji (jizhuozhi.george@gmail.com)
*/
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
#if (NGX_HTTP_UPSTREAM_CHECK)
#include "ngx_http_upstream_check_module.h"
#endif
typedef struct {
ngx_queue_t queue;
ngx_uint_t index;
ngx_uint_t weight;
ngx_uint_t remainder;
ngx_http_upstream_rr_peer_t *peer;
} ngx_http_upstream_iwrr_queue_t;
typedef struct ngx_http_upstream_iwrr_srv_conf_s ngx_http_upstream_iwrr_srv_conf_t;
struct ngx_http_upstream_iwrr_srv_conf_s {
ngx_uint_t init_number;
ngx_http_upstream_iwrr_queue_t *active;
ngx_http_upstream_iwrr_queue_t *expired;
ngx_http_upstream_iwrr_srv_conf_t *next;
};
typedef struct {
/* the round robin data must be first */
ngx_http_upstream_rr_peer_data_t rrp;
ngx_http_upstream_iwrr_srv_conf_t *uiscf;
} ngx_http_upstream_iwrr_peer_data_t;
static char *ngx_http_upstream_iwrr(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf);
static void *ngx_http_upstream_iwrr_create_srv_conf(ngx_conf_t *cf);
static ngx_int_t ngx_http_upstream_init_iwrr(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us);
static ngx_int_t ngx_http_upstream_init_iwrr_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us);
static ngx_int_t ngx_http_upstream_get_iwrr_peer(ngx_peer_connection_t *pc,
void *data);
static ngx_http_upstream_rr_peer_t *ngx_http_upstream_get_iwrr(
ngx_http_upstream_iwrr_peer_data_t *uip);
static ngx_http_upstream_iwrr_queue_t *ngx_http_upstream_iwrr_queue_next(
ngx_http_upstream_iwrr_srv_conf_t *uiscf);
static inline ngx_uint_t ngx_http_upstream_iwrr_gcd(ngx_uint_t a, ngx_uint_t b);
static ngx_command_t ngx_http_upstream_iwrr_commands[] = {
{ ngx_string("iwrr"),
NGX_HTTP_UPS_CONF|NGX_CONF_NOARGS,
ngx_http_upstream_iwrr,
0,
0,
NULL },
ngx_null_command
};
static ngx_http_module_t ngx_http_upstream_iwrr_module_ctx = {
NULL, /* preconfiguration */
NULL, /* postconfiguration */
NULL, /* create main configuration */
NULL, /* init main configuration */
ngx_http_upstream_iwrr_create_srv_conf, /* create server configuration */
NULL, /* merge server configuration */
NULL, /* create location configuration */
NULL /* merge location configuration */
};
ngx_module_t ngx_http_upstream_iwrr_module = {
NGX_MODULE_V1,
&ngx_http_upstream_iwrr_module_ctx, /* module context */
ngx_http_upstream_iwrr_commands, /* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
static char *
ngx_http_upstream_iwrr(ngx_conf_t *cf, ngx_command_t *cmd,
void *conf)
{
ngx_http_upstream_srv_conf_t *uscf;
uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module);
if (uscf->peer.init_upstream) {
ngx_conf_log_error(NGX_LOG_WARN, cf, 0,
"load balancing method redefined");
}
uscf->peer.init_upstream = ngx_http_upstream_init_iwrr;
uscf->flags = NGX_HTTP_UPSTREAM_CREATE
|NGX_HTTP_UPSTREAM_WEIGHT
|NGX_HTTP_UPSTREAM_BACKUP
|NGX_HTTP_UPSTREAM_MAX_FAILS
|NGX_HTTP_UPSTREAM_FAIL_TIMEOUT
#if defined(nginx_version) && nginx_version >= 1011005
|NGX_HTTP_UPSTREAM_MAX_CONNS
#endif
|NGX_HTTP_UPSTREAM_DOWN;
return NGX_CONF_OK;
}
static void *
ngx_http_upstream_iwrr_create_srv_conf(ngx_conf_t *cf)
{
ngx_http_upstream_iwrr_srv_conf_t *uiscf;
uiscf = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_iwrr_srv_conf_t));
if (uiscf == NULL) {
return NULL;
}
/*
* set by ngx_pcalloc():
*
* uiscf->active = NULL;
* uiscf->expired = NULL;
* uiscf->next = NULL;
*/
uiscf->init_number = NGX_CONF_UNSET_UINT;
return uiscf;
}
static ngx_int_t
ngx_http_upstream_init_iwrr(ngx_conf_t *cf,
ngx_http_upstream_srv_conf_t *us)
{
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_iwrr_srv_conf_t *uiscf;
ngx_http_upstream_iwrr_queue_t *item;
ngx_uint_t i, g;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "init iwrr");
if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) {
return NGX_ERROR;
}
us->peer.init = ngx_http_upstream_init_iwrr_peer;
uiscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_iwrr_module);
peers = (ngx_http_upstream_rr_peers_t *) us->peer.data;
for (peers = (ngx_http_upstream_rr_peers_t *) us->peer.data;
peers;
peers = peers->next)
{
g = 0;
for (peer = peers->peer;
peer;
peer = peer->next)
{
g = ngx_http_upstream_iwrr_gcd(g, peer->weight);
}
uiscf->active = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_iwrr_queue_t));
if (uiscf->active == NULL) {
return NGX_ERROR;
}
ngx_queue_init(&uiscf->active->queue);
uiscf->expired = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_iwrr_queue_t));
if (uiscf->active == NULL) {
return NGX_ERROR;
}
ngx_queue_init(&uiscf->expired->queue);
for (peer = peers->peer, i = 0;
peer;
peer = peer->next, i++)
{
item = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_iwrr_queue_t));
if (item == NULL) {
return NGX_ERROR;
}
item->index = i;
item->weight = peer->weight / g;
item->remainder = item->weight;
item->peer = peer;
ngx_queue_insert_tail(&uiscf->active->queue, &item->queue);
}
if (peers->next) {
uiscf->next = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_iwrr_srv_conf_t));
if (uiscf->next == NULL) {
return NGX_ERROR;
}
uiscf->next->init_number = NGX_CONF_UNSET_UINT;
uiscf = uiscf->next;
}
}
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_init_iwrr_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us)
{
ngx_http_upstream_iwrr_srv_conf_t *uiscf;
ngx_http_upstream_iwrr_peer_data_t *uip;
uiscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_iwrr_module);
if (uiscf == NULL) {
return NGX_ERROR;
}
uip = ngx_palloc(r->pool, sizeof(ngx_http_upstream_iwrr_peer_data_t));
if (uip == NULL) {
return NGX_ERROR;
}
uip->uiscf = uiscf;
r->upstream->peer.data = &uip->rrp;
if (ngx_http_upstream_init_round_robin_peer(r, us) != NGX_OK) {
return NGX_ERROR;
}
r->upstream->peer.get = ngx_http_upstream_get_iwrr_peer;
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_get_iwrr_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_iwrr_peer_data_t *uip = data;
ngx_int_t rc;
ngx_uint_t i, n;
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_rr_peer_data_t *rrp;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get iwrr peer, try: %ui", pc->tries);
pc->cached = 0;
pc->connection = NULL;
rrp = &uip->rrp;
peers = rrp->peers;
ngx_http_upstream_rr_peers_wlock(peers);
if (peers->single) {
peer = peers->peer;
if (peer->down) {
goto failed;
}
if (peer->max_conns && peer->conns >= peer->max_conns) {
goto failed;
}
#if (NGX_HTTP_UPSTREAM_CHECK)
if (ngx_http_upstream_check_peer_down(peer->check_index)) {
goto failed;
}
#endif
rrp->current = peer;
} else {
/* there are several peers */
peer = ngx_http_upstream_get_iwrr(uip);
if (peer == NULL) {
goto failed;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get iwrr peer, current: %p %i",
peer, peer->current_weight);
}
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
#if (T_NGX_HTTP_DYNAMIC_RESOLVE)
pc->host = &peer->host;
#endif
peer->conns++;
ngx_http_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
rrp->peers = peers->next;
uip->uiscf = uip->uiscf ? uip->uiscf->next : uip->uiscf;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++) {
rrp->tried[i] = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
rc = ngx_http_upstream_get_iwrr_peer(pc, uip);
if (rc != NGX_BUSY) {
return rc;
}
ngx_http_upstream_rr_peers_wlock(peers);
}
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
static ngx_http_upstream_rr_peer_t *
ngx_http_upstream_get_iwrr(ngx_http_upstream_iwrr_peer_data_t *uip)
{
time_t now;
uintptr_t m;
ngx_uint_t i, j, n;
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_rr_peers_t *peers;
ngx_http_upstream_rr_peer_data_t *rrp;
ngx_http_upstream_iwrr_srv_conf_t *uiscf;
ngx_http_upstream_iwrr_queue_t *item;
now = ngx_time();
rrp = &uip->rrp;
peers = rrp->peers;
uiscf = uip->uiscf;
#if (T_NGX_HTTP_UPSTREAM_RANDOM)
if (uiscf->init_number == NGX_CONF_UNSET_UINT) {
uiscf->init_number = ngx_random() % peers->number;
for (i = 0; i < uiscf->init_number; i++) {
ngx_http_upstream_iwrr_queue_next(uiscf);
}
}
#endif
for (j = 0; j < peers->number; j++) {
item = ngx_http_upstream_iwrr_queue_next(uiscf);
i = item->index;
peer = item->peer;
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m) {
continue;
}
if (peer->down) {
continue;
}
#if (NGX_HTTP_UPSTREAM_CHECK)
if (ngx_http_upstream_check_peer_down(peer->check_index)) {
continue;
}
#endif
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
if (peer->max_conns && peer->conns >= peer->max_conns) {
continue;
}
rrp->current = peer;
return peer;
}
return NULL;
}
static ngx_http_upstream_iwrr_queue_t *
ngx_http_upstream_iwrr_queue_next(ngx_http_upstream_iwrr_srv_conf_t *uiscf)
{
ngx_http_upstream_iwrr_queue_t *temp, *item;
if (ngx_queue_empty(&uiscf->active->queue)) {
temp = uiscf->active;
uiscf->active = uiscf->expired;
uiscf->expired = temp;
}
item = (ngx_http_upstream_iwrr_queue_t *) ngx_queue_head(&uiscf->active->queue);
ngx_queue_remove(&item->queue);
item->remainder--;
if (item->remainder) {
ngx_queue_insert_tail(&uiscf->active->queue, &item->queue);
} else {
item->remainder = item->weight;
ngx_queue_insert_tail(&uiscf->expired->queue, &item->queue);
}
return item;
}
static inline ngx_uint_t ngx_http_upstream_iwrr_gcd(ngx_uint_t a, ngx_uint_t b)
{
ngx_uint_t r;
while (b) {
r = a % b;
a = b;
b = r;
}
return a;
}

234
tests/nginx-tests/tengine-tests/iwrr.t
View File

@ -0,0 +1,234 @@
#!/usr/bin/perl
# Copyright (C) 2010-2023 Alibaba Group Holding Limited
# Copyright (C) 2010-2023 Zhuozhi Ji (jizhuozhi.george@gmail.com)
# Tests for upstream iwrr balancer module.
###############################################################################
use warnings;
use Test::More;
BEGIN { use FindBin; chdir($FindBin::Bin); }
use lib 'lib';
use Test::Nginx qw/ :DEFAULT :gzip/ ;
###############################################################################
select STDERR; $| = 1;
select STDOUT; $| = 1;
my $t = Test::Nginx->new()->has(qw/http proxy upstream_zone iwrr/)
->write_file_expand('nginx.conf', <<'EOF');
%%TEST_GLOBALS%%
daemon off;
worker_processes 1;
events {
}
http {
%%TEST_GLOBALS_HTTP%%
upstream u {
iwrr;
server 127.0.0.1:8081;
server 127.0.0.1:8082;
server 127.0.0.1:8083 down;
}
upstream w {
iwrr;
server 127.0.0.1:8081;
server 127.0.0.1:8082 weight=2;
}
upstream zone {
iwrr;
server 127.0.0.1:8081;
}
upstream b {
iwrr;
server 127.0.0.1:8081 down;
server 127.0.0.1:8082 backup;
}
upstream d {
iwrr;
server 127.0.0.1:8081;
server 127.0.0.1:8082;
server 127.0.0.1:8083 weight=2;
server 127.0.0.1:8084 down;
}
upstream g {
iwrr;
server 127.0.0.1:8081 weight=2;
server 127.0.0.1:8082 weight=4;
server 127.0.0.1:8083 weight=8;
}
upstream h {
iwrr;
server 127.0.0.1:8081;
server 127.0.0.1:8082;
server 127.0.0.1:8083;
}
upstream i {
iwrr;
server 127.0.0.1:8081 down;
server 127.0.0.1:8082 backup;
server 127.0.0.1:8083 backup;
}
server {
listen 127.0.0.1:8081;
listen 127.0.0.1:8082;
listen 127.0.0.1:8083;
server_name localhost;
location / {
return 200 $server_port;
}
}
server {
listen 127.0.0.1:8080;
server_name localhost;
location / {
proxy_pass http://u;
}
location /w {
proxy_pass http://w;
}
location /zone {
proxy_pass http://zone;
}
location /b {
proxy_pass http://b;
}
location /d {
proxy_pass http://d;
}
location /g {
proxy_pass http://g;
}
location /h {
proxy_pass http://h;
}
location /i {
proxy_pass http://i;
}
}
}
EOF
$t->try_run('no upstream iwrr')->plan(18);
###############################################################################
my $r;
my %list = ();
$list{'8083'} = 0;
$list{http_get_body('/')} = 1;
$list{http_get_body('/')} = 1;
$list{http_get_body('/')} = 1;
is($list{'8081'}, 1, 'iwrr 8081');
is($list{'8082'}, 1, 'iwrr 8082');
is($list{'8083'}, 0, 'peer down');
%list = ();
$list{http_get_body('/w')} += 1;
$list{http_get_body('/w')} += 1;
$list{http_get_body('/w')} += 1;
is($list{'8081'}, 1, 'weight 1');
is($list{'8082'}, 2, 'weight 2');
%list = ();
$list{http_get_body('/zone')} += 1;
is($list{'8081'}, 1, 'iwrr zone');
%list = ();
$list{http_get_body('/b')} += 1;
is($list{'8082'}, 1, 'iwrr backup');
%list = ();
$list{http_get_body('/d')} += 1;
$list{http_get_body('/d')} += 1;
$list{http_get_body('/d')} += 1;
$list{http_get_body('/d')} += 1;
is($list{'8081'}, 1, 'weight 1');
is($list{'8082'}, 1, 'weight 1');
is($list{'8083'}, 2, 'weight 2');
%list = ();
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
$list{http_get_body('/g')} += 1;
is($list{'8081'}, 2, 'weight 2');
is($list{'8082'}, 4, 'weight 4');
is($list{'8083'}, 8, 'weight 8');
%list = ();
$list{http_get_body('/h')} += 1;
$list{http_get_body('/h')} += 1;
$list{http_get_body('/h')} += 1;
is($list{'8081'}, 1, 'weight 1');
is($list{'8082'}, 1, 'weight 1');
is($list{'8083'}, 1, 'weight 1');
%list = ();
$list{http_get_body('/i')} += 1;
$list{http_get_body('/i')} += 1;
is($list{'8082'}, 1, 'weight 1');
is($list{'8083'}, 1, 'weight 1');
###############################################################################
sub http_get_body {
my ($uri) = @_;
return undef if !defined $uri;
http_get($uri) =~ /(.*?)\x0d\x0a?\x0d\x0a?(.*)/ms;
return $2;
}
###############################################################################
Write Preview
Loading…
Cancel
Save