[大数据]bitmap过滤海量重复数据

需求

从海量的帖子中推荐一页帖子在首页，尽量避免用户刷到相同的帖子；

需求分析

1.针对用户推荐帖子
2.避免重复数据
3.帖子数据量巨大

实现

1.考虑用redis 布隆过滤器，但是因为我们可以得到帖子id是唯一的，直接用id作为hash值写入redis的bitmap ;
我这边是用户只要两个月不需要看到重复的数据即可；如果是没有时间限制则去掉对应的时间限制

RedisAbstract 缓存抽象类

abstract class RedisAbstract
{
    protected $prefix = 'rds';

    protected $name = 'default';

    /**
     * 获取 Redis 连接
     * @return RedisData|mixed
     */
    protected function redis()
    {
        return \Yii::$app->redisData;
    }

    /**
     * 获取缓存 KEY
     *
     * @param string|array $key
     * @return string
     */
    protected function getCacheKey($key = '')
    {
        $params = [$this->prefix, $this->name];
        if (is_array($key)) {
            $params = array_merge($params, $key);
        } else {
            $params[] = $key;
        }

        return $this->filter($params);
    }

    /**
     * @param array $params
     * @return string
     */
    protected function filter(array $params = [])
    {
        foreach ($params as $k => $param) {
            $params[$k] = trim($param, ':');
        }

        return implode(':', array_filter($params));
    }

}

FilterRepeated 过滤器

class FilterRepeated extends RedisAbstract
{
    protected $month;
    protected $userId;
    protected $name = "post";

    protected $bucket;
    protected $prefix = 'bl';

    protected function __construct($userId, $month)
    {
        $this->userId = $userId;
        $this->month = $month;
        $keyArr = [$userId, $this->month];
        $this->bucket = $this->getCacheKey($keyArr);
    }
    
    /**
     * 实例化当月对象
     * @param $userId
     * @return FilterRepeated
     */
    public static function fromThisMonth($userId)
    {
        return new self($userId, date('y_m'));
    }

    /**
     * 实例上月对象
     * @param $userId
     * @return FilterRepeated
     */
    public static function fromLastMonth($userId)
    {
        return new self($userId, date('y_m', strtotime('-1 month')));
    }

    /**
     * userId 是否看过ID
     * @param $userId
     * @param $string
     * @return bool
     */
    public static function existsLastTwoMonth($userId, $string)
    {
        $flag = false;
        //当月是否有看过
        $thisMonth = self::fromThisMonth($userId);
        if ($thisMonth->exists($string)) {
            $flag = true;
        }
        //没看过的话，上月是否看过
        if (!$flag) {
            $lastMonthFilter = self::fromLastMonth($userId);
            if ($lastMonthFilter->exists($string)) {
                $flag = true;
            }
        }

        return $flag;
    }

    /**
     * @param int $id
     * @return mixed
     */
    public function add(int $id)
    {
        return $this->redis()->setbit($this->bucket, $id, 1);
    }

    /**
     * @param array $ids
     * @return mixed
     */
    public function addMulti(array $ids)
    {
        if (count($ids) > 0) {
            $this->redis()->multi();
            foreach ($ids as $id) {
                $this->redis()->setbit($this->bucket, $id, 1);
            }
            return $this->redis()->exec();
        }
    }

    /**
     * @param $postId
     * @return bool
     */
    public function exists($id)
    {
        $bit = $this->redis()->getbit($this->bucket, $id);
        if ($bit == 0) {
            return false;
        }
        return true;
    }

    /**
     * 返回多个验证结果
     * @param $ids
     * @return mixed
     */
    public function existMulti($ids)
    {
        $this->redis()->multi();
        foreach ($ids as $id) {
            $this->redis()->getbit($this->bucket, $id);
        }
        return  $this->redis()->exec();
    }

    /**
     * bucket 下所有的数据清除
     */
    public function flush()
    {
        $this->redis()->expire($this->bucket, 0);
    }
}

创作打卡挑战赛

赢取流量/现金/CSDN周边激励大奖